lhm-shopify 4.4.0 → 4.4.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 9b9ff81249ed6289557c131a6b6183dd77b494af67b71cb4845e69628092801e
-  data.tar.gz: ca4a71a59985f62685056833ad288a63ff33b0ca233a7a41571cb7f0963b096e
+  metadata.gz: 0ef875eccb1a986bb82ef5ee9d981e229ca54fc92e58f61b6e48768928e48baf
+  data.tar.gz: d206155bcf16a94c2dc78622826c35b57f7e09b4d9fff1ddad9726d1a48d61fa
 SHA512:
-  metadata.gz: ae9eedad45c7425785150927537a604084c60215c1bdfa796e7bc018fdddbaff2e431af97f5611d15ad5064ad9640f4af38c24f44712eb3567a66bba9f128038
-  data.tar.gz: 67135252c34441946ff069a1c6283545656a60b77306f8e6c019d319f0a164fd17dd898a21fcbfeba5490a52598812a804de52ad5fa498b6eee77a7731bc49ae
+  metadata.gz: 6f3c37c8387638aebcdcb3caced88a39b998247a55a130fc8eade81b4c37622af8acdb570e234ce1c3fa25562ec460844e9cc48e9d9d5d1fe4284c9a6d7f4be8
+  data.tar.gz: d5a1b197b4260aa494a30de528cdf6e5cfa6e8b81c9f40b40eb9e71fcdcae1c73db6f955d599d5930d581b7c1be7623dc0f96e0112073aa063f178157b1b52c6

data/CHANGELOG.md

@@ -1,5 +1,8 @@
 # Unreleased
 
+# 4.4.1 (Aug, 2024)
+* Extend max_binlog_cache_size exceeded error handling to all throttlers
+
 # 4.4.0 (Aug, 2024)
 * Add support for retrying chunks when running into max_binlog_cache_size exceeded error
 

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    lhm-shopify (4.4.0)
+    lhm-shopify (4.4.1)
       retriable (>= 3.0.0)
 
 GEM

data/gemfiles/activerecord_6.1.gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: ..
   specs:
-    lhm-shopify (4.4.0)
+    lhm-shopify (4.4.1)
       retriable (>= 3.0.0)
 
 GEM

data/gemfiles/activerecord_7.0.gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: ..
   specs:
-    lhm-shopify (4.4.0)
+    lhm-shopify (4.4.1)
       retriable (>= 3.0.0)
 
 GEM

data/gemfiles/activerecord_7.1.gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: ..
   specs:
-    lhm-shopify (4.4.0)
+    lhm-shopify (4.4.1)
       retriable (>= 3.0.0)
 
 GEM

data/gemfiles/activerecord_head.gemfile.lock

@@ -24,7 +24,7 @@ GIT
 PATH
   remote: ..
   specs:
-    lhm-shopify (4.4.0)
+    lhm-shopify (4.4.1)
       retriable (>= 3.0.0)
 
 GEM

data/lib/lhm/chunker.rb

@@ -37,10 +37,6 @@ module Lhm
       )
     end
 
-    def handle_max_binlog_exceeded_error
-      @throttler.backoff_stride
-    end
-
     def execute
       @start_time = Time.now
 
@@ -54,9 +50,9 @@ module Lhm
         begin
           affected_rows = ChunkInsert.new(@migration, @connection, bottom, top, @retry_options).insert_and_return_count_of_rows_created
         rescue ActiveRecord::StatementInvalid => e
-          if e.message.downcase.include?("transaction required more than 'max_binlog_cache_size' bytes of storage")
+          if e.message.downcase.include?("transaction required more than 'max_binlog_cache_size' bytes of storage") && @throttler.respond_to?(:backoff_stride)
             Lhm.logger.info("Encountered max_binlog_cache_size error, attempting to reduce stride size")
-            handle_max_binlog_exceeded_error
+            @throttler.backoff_stride
             next
           else
             raise e

data/lib/lhm/throttler/backoff_reduction.rb

@@ -0,0 +1,42 @@
+module Lhm
+  module Throttler
+    module BackoffReduction
+      DEFAULT_BACKOFF_REDUCTION_FACTOR = 0.2
+      MIN_STRIDE_SIZE = 1
+
+      def initialize(options = {})
+        @backoff_reduction_factor = options[:backoff_reduction_factor] || DEFAULT_BACKOFF_REDUCTION_FACTOR
+        @min_stride_size = options[:min_stride_size] || MIN_STRIDE_SIZE
+
+        if @backoff_reduction_factor >= 1 || @backoff_reduction_factor <= 0
+          raise ArgumentError, 'backoff_reduction_factor must be between greater than 0, and less than 1'
+        end
+
+        if @min_stride_size < 1
+          raise ArgumentError, 'min_stride_size must be an integer greater than 0'
+        end
+
+        if !@min_stride_size.is_a?(Integer)
+          raise ArgumentError, 'min_stride_size must be an integer'
+        end
+
+        if @min_stride_size > @stride
+          raise ArgumentError, 'min_stride_size must be less than or equal to stride'
+        end
+      end
+
+      def backoff_stride
+        new_stride = (@stride * (1 - @backoff_reduction_factor)).to_i
+
+        if new_stride == @stride
+          raise RuntimeError, "Cannot backoff any further"
+        end
+
+        if new_stride < @min_stride_size
+          raise RuntimeError, "Cannot reduce stride below #{@min_stride_size}"
+        end
+        @stride = new_stride
+      end
+    end
+  end
+end

data/lib/lhm/throttler/replica_lag.rb

@@ -13,6 +13,7 @@ module Lhm
 
     class ReplicaLag
       include Command
+      include BackoffReduction
 
       INITIAL_TIMEOUT = 0.1
       DEFAULT_STRIDE = 2_000
@@ -29,6 +30,8 @@ module Lhm
         @replicas = {}
         @get_config = options[:current_config]
         @check_only = options[:check_only]
+
+        super
       end
 
       def execute

data/lib/lhm/throttler/threads_running.rb

@@ -2,6 +2,7 @@ module Lhm
   module Throttler
     class ThreadsRunning
       include Command
+      include BackoffReduction
 
       DEFAULT_STRIDE = 2_000
       DEFAULT_INITIAL_TIMEOUT = 0.1
@@ -17,6 +18,8 @@ module Lhm
         @timeout_seconds = @initial_timeout_seconds
         @healthy_range = options[:healthy_range] || DEFAULT_HEALTHY_RANGE
         @connection = options[:connection]
+
+        super
       end
 
       def threads_running

data/lib/lhm/throttler/time.rb

@@ -2,6 +2,7 @@ module Lhm
   module Throttler
     class Time
       include Command
+      include BackoffReduction
 
       DEFAULT_TIMEOUT = 0.1
       DEFAULT_STRIDE = 2_000
@@ -14,37 +15,8 @@ module Lhm
       def initialize(options = {})
         @timeout_seconds = options[:delay] || DEFAULT_TIMEOUT
         @stride = options[:stride] || DEFAULT_STRIDE
-        @backoff_reduction_factor = options[:backoff_reduction_factor] || DEFAULT_BACKOFF_REDUCTION_FACTOR
-        @min_stride_size = options[:min_stride_size] || MIN_STRIDE_SIZE
 
-        if @backoff_reduction_factor >= 1 || @backoff_reduction_factor <= 0
-          raise ArgumentError, 'backoff_reduction_factor must be between greater than 0, and less than 1'
-        end
-
-        if @min_stride_size < 1
-          raise ArgumentError, 'min_stride_size must be an integer greater than 0'
-        end
-
-        if !@min_stride_size.is_a?(Integer)
-          raise ArgumentError, 'min_stride_size must be an integer'
-        end
-
-        if @min_stride_size > @stride
-          raise ArgumentError, 'min_stride_size must be less than or equal to stride'
-        end
-      end
-
-      def backoff_stride
-        new_stride = (@stride * (1 - @backoff_reduction_factor)).to_i
-
-        if new_stride == @stride
-          raise RuntimeError, "Cannot backoff any further"
-        end
-
-        if new_stride < @min_stride_size
-          raise RuntimeError, "Cannot reduce stride below #{@min_stride_size}"
-        end
-        @stride = new_stride
+        super
       end
 
       def execute

data/lib/lhm/throttler.rb

@@ -1,3 +1,4 @@
+require 'lhm/throttler/backoff_reduction'
 require 'lhm/throttler/time'
 require 'lhm/throttler/replica_lag'
 require 'lhm/throttler/slave_lag'

data/lib/lhm/version.rb

@@ -2,5 +2,5 @@
 # Schmidt
 
 module Lhm
-  VERSION = '4.4.0'
+  VERSION = '4.4.1'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: lhm-shopify
 version: !ruby/object:Gem::Version
-  version: 4.4.0
+  version: 4.4.1
 platform: ruby
 authors:
 - SoundCloud
@@ -12,7 +12,7 @@ authors:
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2024-08-20 00:00:00.000000000 Z
+date: 2024-08-21 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: retriable
@@ -239,6 +239,7 @@ files:
 - lib/lhm/table_name.rb
 - lib/lhm/test_support.rb
 - lib/lhm/throttler.rb
+- lib/lhm/throttler/backoff_reduction.rb
 - lib/lhm/throttler/replica_lag.rb
 - lib/lhm/throttler/slave_lag.rb
 - lib/lhm/throttler/threads_running.rb