lhm-shopify 3.5.4 → 3.5.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 4f676427b10052113b08c4dda8f648bc11234024deab2f1e96de332e0906a110
-  data.tar.gz: dbd91d1b72d713e9d6e18edb46900971ee89c20563d5e93db9eec746887a12d7
+  metadata.gz: c3893b743c675e62933e58a4f56c0ac2dba1b1081afabc779dc1d899406e7c42
+  data.tar.gz: 569175938b8069e0036f881400b9427180283ec61c29ca5d509a7090a7f8a9ba
 SHA512:
-  metadata.gz: be5dc9f5d6d2a3ae62b476fa19455bd0f864b989d6ffbff35a93a0afea4e8d9be4492a4b3bfc2bec1029193114187eaf4eda5bb6dfde9d0be34884bde317c1c1
-  data.tar.gz: aa4e866f74da3255586c1597ae63a3792606bd092e46c3a90335d22b9f6c9b243bc133d99e50233c884b03064dfaeee75983597f84733484e05ec456833f3395
+  metadata.gz: ba3ebf953f97cd793c52cb83fee77dd4436877ca01a51ee6d718c7937916e391a32c659b9f403b62f65cf0a326dc755e6221e3311760738c9c6fb442746e1617
+  data.tar.gz: 2e617329a895d00a2541539526e941fc94dfda8285083d46d4179587a31786cbbb1062bd8dbdfffcc4d046801944ac96aa700c32618a4db9f4346345e9105c7c

data/CHANGELOG.md

@@ -1,3 +1,6 @@
+# 3.5.5 (Jan, 2022)
+* Fix error where from Config shadowing which would cause LHM to abort on reconnect (https://github.com/Shopify/lhm/pull/128)
+
 # 3.5.4 (Dec, 2021)
 * Refactored the way options are handled internally. Code is now much clearer to understand
 * Removed optional connection_options from `Lhm.setup` and `Lhm.connection`

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    lhm-shopify (3.5.4)
+    lhm-shopify (3.5.5)
       retriable (>= 3.0.0)
 
 GEM

data/dev.yml

@@ -40,3 +40,6 @@ commands:
     run: docker-compose logs -f
   clear:
     run: docker-compose rm -v -s -f && docker-compose up -d && ./scripts/helpers/wait-for-dbs.sh
+  pre-publish:
+    # Ensures all Gemfile.lock are sync with the new version in `lhm/version.rb` and runs appraisals
+    run: bundle install && bundle exec appraisal install && bundle exec appraisal rake specs

data/gemfiles/activerecord_5.2.gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: ..
   specs:
-    lhm-shopify (3.5.4)
+    lhm-shopify (3.5.5)
       retriable (>= 3.0.0)
 
 GEM

data/gemfiles/activerecord_6.0.gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: ..
   specs:
-    lhm-shopify (3.5.4)
+    lhm-shopify (3.5.5)
       retriable (>= 3.0.0)
 
 GEM

data/gemfiles/activerecord_6.1.gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: ..
   specs:
-    lhm-shopify (3.5.4)
+    lhm-shopify (3.5.5)
       retriable (>= 3.0.0)
 
 GEM

data/gemfiles/activerecord_7.0.0.alpha2.gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: ..
   specs:
-    lhm-shopify (3.5.4)
+    lhm-shopify (3.5.5)
       retriable (>= 3.0.0)
 
 GEM

data/lib/lhm/sql_retry.rb

@@ -18,6 +18,12 @@ module Lhm
   class SqlRetry
     RECONNECT_SUCCESSFUL_MESSAGE = "LHM successfully reconnected to initial host:"
     CLOUDSQL_VERSION_COMMENT = "(Google)"
+    # Will retry for 120 seconds (approximately, since connecting takes time).
+    RECONNECT_RETRY_MAX_ITERATION = 120
+    RECONNECT_RETRY_INTERVAL = 1
+    # Will abort the LHM if it had to reconnect more than 25 times in a single run (indicator that there might be
+    # something wrong with the network and would be better to run the LHM at a later time).
+    RECONNECTION_MAXIMUM = 25
 
     MYSQL_VAR_NAMES = {
       hostname: "@@global.hostname",
@@ -25,9 +31,6 @@ module Lhm
       version_comment: "@@version_comment",
     }
 
-    # This internal error is used to trigger retries from the parent Retriable.retriable in #with_retries
-    class ReconnectToHostSuccessful < Lhm::Error; end
-
     def initialize(connection, retry_options: {}, reconnect_with_consistent_host: false)
       @connection = connection
       self.retry_config = retry_options
@@ -38,11 +41,14 @@ module Lhm
     def with_retries(log_prefix: nil)
       @log_prefix = log_prefix || "" # No prefix. Just logs
 
+      # Amount of time LHM had to reconnect. Aborting if more than RECONNECTION_MAXIMUM
+      reconnection_counter = 0
+
       Retriable.retriable(@retry_config) do
         # Using begin -> rescue -> end for Ruby 2.4 compatibility
         begin
           if @reconnect_with_consistent_host
-            raise Lhm::Error.new("Could not reconnected to initial MySQL host. Aborting to avoid data-loss") unless same_host_as_initial?
+            raise Lhm::Error.new("MySQL host has changed since the start of the LHM. Aborting to avoid data-loss") unless same_host_as_initial?
           end
 
           yield(@connection)
@@ -51,7 +57,18 @@ module Lhm
           # The error will be raised the connection is still active (i.e. no need to reconnect) or if the connection is
           # dead (i.e. not active) and @reconnect_with_host is false (i.e. instructed not to reconnect)
           raise e if @connection.active? || (!@connection.active? && !@reconnect_with_consistent_host)
-          reconnect_with_host_check! if @reconnect_with_consistent_host
+
+          # Lhm could be stuck in a weird state where it loses connection, reconnects and re looses-connection instantly
+          # after, creating an infinite loop (because of the usage of `retry`). Hence, abort after 25 reconnections
+          raise Lhm::Error.new("LHM reached host reconnection max of #{RECONNECTION_MAXIMUM} times. " \
+            "Please try again later.") if reconnection_counter > RECONNECTION_MAXIMUM
+
+          reconnection_counter += 1
+          if reconnect_with_host_check!
+            retry
+          else
+            raise Lhm::Error.new("LHM tried the reconnection procedure but failed. Aborting")
+          end
         end
       end
     end
@@ -105,34 +122,32 @@ module Lhm
 
     def reconnect_with_host_check!
       log_with_prefix("Lost connection to MySQL, will retry to connect to same host")
-      begin
-        Retriable.retriable(host_retry_config) do
+
+      RECONNECT_RETRY_MAX_ITERATION.times do
+        begin
+          sleep(RECONNECT_RETRY_INTERVAL)
+
           # tries to reconnect. On failure will trigger a retry
           @connection.reconnect!
 
           if same_host_as_initial?
             # This is not an actual error, but controlled way to get the parent `Retriable.retriable` to retry
             # the statement that failed (since the Retriable gem only retries on errors).
-            raise ReconnectToHostSuccessful.new("LHM successfully reconnected to initial host: #{@initial_hostname} (server_id: #{@initial_server_id})")
+            log_with_prefix("LHM successfully reconnected to initial host: #{@initial_hostname} (server_id: #{@initial_server_id})")
+            return true
           else
             # New Master --> abort LHM (reconnecting will not change anything)
-            raise Lhm::Error.new("Reconnected to wrong host. Started migration on: #{@initial_hostname} (server_id: #{@initial_server_id}), but reconnected to: #{hostname} (server_id: #{@initial_server_id}).")
+            log_with_prefix("Reconnected to wrong host. Started migration on: #{@initial_hostname} (server_id: #{@initial_server_id}), but reconnected to: #{hostname} (server_id: #{server_id}).", :error)
+            return false
           end
+        rescue StandardError => e
+          # Retry if ActiveRecord cannot reach host
+          next if /Lost connection to MySQL server at 'reading initial communication packet'/ === e.message
+          log_with_prefix("Encountered error: [#{e.class}] #{e.message}. Will stop reconnection procedure.", :info)
+          return false
         end
-      rescue StandardError => e
-        # The parent Retriable.retriable is configured to retry if it encounters an error with the success message.
-        # Therefore, if the connection is re-established successfully AND the host is the same, LHM can retry the query
-        # that originally failed.
-        raise e if reconnect_successful?(e)
-        # If the connection was not successful, the parent retriable will raise any error that originated from the
-        # `@connection.reconnect!`
-        # Therefore, this error will cause the LHM to abort
-        raise Lhm::Error.new("LHM tried the reconnection procedure but failed. Latest error: #{e.message}")
       end
-    end
-
-    def reconnect_successful?(e)
-      e.is_a?(ReconnectToHostSuccessful)
+      false
     end
 
     # For a full list of configuration options see https://github.com/kamui/retriable
@@ -149,9 +164,6 @@ module Lhm
             /Unknown MySQL server host/,
             /connection is locked to hostgroup/,
             /The MySQL server is running with the --read-only option so it cannot execute this statement/,
-          ],
-          ReconnectToHostSuccessful => [
-            /#{RECONNECT_SUCCESSFUL_MESSAGE}/
           ]
         },
         multiplier: 1, # each successive interval grows by this factor
@@ -159,28 +171,6 @@ module Lhm
         tries: 20, # Number of attempts to make at running your code block (includes initial attempt).
         rand_factor: 0, # percentage to randomize the next retry interval time
         max_elapsed_time: Float::INFINITY, # max total time in seconds that code is allowed to keep being retried
-        on_retry: Proc.new do |exception, try_number, total_elapsed_time, next_interval|
-          if reconnect_successful?(exception)
-            log_with_prefix("#{exception.message} -- triggering retry", :info)
-          else
-            log_with_prefix("#{exception.class}: '#{exception.message}' - #{try_number} tries in #{total_elapsed_time} seconds and #{next_interval} seconds until the next try.", :error)
-          end
-        end
-      }.freeze
-    end
-
-    def host_retry_config
-      {
-        on: {
-          StandardError => [
-            /Lost connection to MySQL server at 'reading initial communication packet'/
-          ]
-        },
-        multiplier: 1, # each successive interval grows by this factor
-        base_interval: 0.25, # the initial interval in seconds between tries.
-        tries: 20, # Number of attempts to make at running your code block (includes initial attempt).
-        rand_factor: 0, # percentage to randomize the next retry interval time
-        max_elapsed_time: Float::INFINITY, # max total time in seconds that code is allowed to keep being retried
         on_retry: Proc.new do |exception, try_number, total_elapsed_time, next_interval|
           log_with_prefix("#{exception.class}: '#{exception.message}' - #{try_number} tries in #{total_elapsed_time} seconds and #{next_interval} seconds until the next try.", :error)
         end

data/lib/lhm/version.rb

@@ -2,5 +2,5 @@
 # Schmidt
 
 module Lhm
-  VERSION = '3.5.4'
+  VERSION = '3.5.5'
 end

data/spec/integration/lhm_spec.rb

@@ -645,7 +645,6 @@ describe Lhm do
         log_lines = @logs.string.split("\n")
 
         assert log_lines.one?{ |line| line.include?("Lost connection to MySQL, will retry to connect to same host")}
-        assert log_lines.any?{ |line| line.include?("Lost connection to MySQL server at 'reading initial communication packet'")}
         assert log_lines.one?{ |line| line.include?("LHM successfully reconnected to initial host")}
         assert log_lines.one?{ |line| line.include?("100% complete")}
 

data/spec/integration/sql_retry/retry_with_proxysql_spec.rb

@@ -38,7 +38,7 @@ describe Lhm::SqlRetry, "ProxiSQL tests for LHM retry" do
       end
     end
     assert_equal Lhm::Error, e.class
-    assert_match(/LHM tried the reconnection procedure but failed. Latest error:/, e.message)
+    assert_match(/LHM tried the reconnection procedure but failed. Aborting/, e.message)
   end
 
   it "Will retry until connection is achieved" do
@@ -61,6 +61,7 @@ describe Lhm::SqlRetry, "ProxiSQL tests for LHM retry" do
   it "Will abort if new writer is not same host" do
     # The hostname will be constant before the blip
     Lhm::SqlRetry.any_instance.stubs(:hostname).returns("mysql-1").then.returns("mysql-2")
+    Lhm::SqlRetry.any_instance.stubs(:server_id).returns(1).then.returns(2)
 
     # Need new instance for stub to take into effect
     lhm_retry = Lhm::SqlRetry.new(@connection, retry_options: {},
@@ -76,12 +77,12 @@ describe Lhm::SqlRetry, "ProxiSQL tests for LHM retry" do
     end
 
     assert_equal e.class, Lhm::Error
-    assert_match(/LHM tried the reconnection procedure but failed. Latest error: Reconnected to wrong host/, e.message)
+    assert_match(/LHM tried the reconnection procedure but failed. Aborting/, e.message)
 
     logs = @logger.string.split("\n")
 
     assert logs.first.include?("Lost connection to MySQL, will retry to connect to same host")
-    assert logs.last.include?("Lost connection to MySQL server at 'reading initial communication packet")
+    assert logs.last.include?("Reconnected to wrong host. Started migration on: mysql-1 (server_id: 1), but reconnected to: mysql-2 (server_id: 2).")
   end
 
   it "Will abort if failover happens (mimicked with proxySQL)" do
@@ -98,11 +99,11 @@ describe Lhm::SqlRetry, "ProxiSQL tests for LHM retry" do
     end
 
     assert_equal e.class, Lhm::Error
-    assert_match(/LHM tried the reconnection procedure but failed. Latest error: Reconnected to wrong host/, e.message)
+    assert_match(/LHM tried the reconnection procedure but failed. Aborting/, e.message)
 
     logs = @logger.string.split("\n")
 
     assert logs.first.include?("Lost connection to MySQL, will retry to connect to same host")
-    assert logs.last.include?("Lost connection to MySQL server at 'reading initial communication packet")
+    assert logs.last.include?("Reconnected to wrong host. Started migration on: mysql-1 (server_id: 1), but reconnected to: mysql-2 (server_id: 2).")
   end
 end

data/spec/test_helper.rb

@@ -28,6 +28,9 @@ logger = Logger.new STDOUT
 logger.level = Logger::WARN
 Lhm.logger = logger
 
+# Want test to be efficient without having to wait the normal value of 120s
+Lhm::SqlRetry::RECONNECT_RETRY_MAX_ITERATION = 4
+
 def without_verbose(&block)
   old_verbose, $VERBOSE = $VERBOSE, nil
   yield

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: lhm-shopify
 version: !ruby/object:Gem::Version
-  version: 3.5.4
+  version: 3.5.5
 platform: ruby
 authors:
 - SoundCloud
@@ -12,7 +12,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2021-12-14 00:00:00.000000000 Z
+date: 2022-01-07 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: retriable