lhm-shopify 4.3.0 → 4.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: f0c79dc83baf664e8cd4ad2e11f8e72b3927edbb2af0beb5626961339e7f75c7
-  data.tar.gz: 1a58b01ab3ab768c37f497e24329d0e4e3d85dc64af7b1a338c5447b742f4a4b
+  metadata.gz: 9b9ff81249ed6289557c131a6b6183dd77b494af67b71cb4845e69628092801e
+  data.tar.gz: ca4a71a59985f62685056833ad288a63ff33b0ca233a7a41571cb7f0963b096e
 SHA512:
-  metadata.gz: e349310577754cebfe7fa4d067727946c19fb0b291f1f2305e87d26a7d30371fd37c80bcdb58039936c90c75618157904f3141722e5f3f83c9b0e05afb76e447
-  data.tar.gz: 63fe5f4a016b32ab3d299e5cd9368f226a575bd725040931b8dd28db7e8a35cef7fbd78f0ec2713108dd508a974c7b783734a02e4d724a3b40e00afb7502693e
+  metadata.gz: ae9eedad45c7425785150927537a604084c60215c1bdfa796e7bc018fdddbaff2e431af97f5611d15ad5064ad9640f4af38c24f44712eb3567a66bba9f128038
+  data.tar.gz: 67135252c34441946ff069a1c6283545656a60b77306f8e6c019d319f0a164fd17dd898a21fcbfeba5490a52598812a804de52ad5fa498b6eee77a7731bc49ae

data/Appraisals

@@ -11,3 +11,7 @@ end
 appraise "activerecord-7.1" do
   gem "activerecord", "7.1.1"
 end
+
+appraise "activerecord-head" do
+  gem "activerecord", git: "https://github.com/rails/rails.git", branch: "main"
+end

data/CHANGELOG.md

@@ -1,5 +1,8 @@
 # Unreleased
 
+# 4.4.0 (Aug, 2024)
+* Add support for retrying chunks when running into max_binlog_cache_size exceeded error
+
 # 4.3.0 (Aug, 2024)
 * Drop support for Ruby 3.0, as it reached its EOL
 * Add support for next Rails version

data/Gemfile.lock

@@ -1,28 +1,29 @@
 PATH
   remote: .
   specs:
-    lhm-shopify (4.3.0)
+    lhm-shopify (4.4.0)
       retriable (>= 3.0.0)
 
 GEM
   remote: https://rubygems.org/
   specs:
-    activemodel (7.1.3.4)
-      activesupport (= 7.1.3.4)
-    activerecord (7.1.3.4)
-      activemodel (= 7.1.3.4)
-      activesupport (= 7.1.3.4)
+    activemodel (7.2.0)
+      activesupport (= 7.2.0)
+    activerecord (7.2.0)
+      activemodel (= 7.2.0)
+      activesupport (= 7.2.0)
       timeout (>= 0.4.0)
-    activesupport (7.1.3.4)
+    activesupport (7.2.0)
       base64
       bigdecimal
-      concurrent-ruby (~> 1.0, >= 1.0.2)
+      concurrent-ruby (~> 1.0, >= 1.3.1)
       connection_pool (>= 2.2.5)
       drb
       i18n (>= 1.6, < 2)
+      logger (>= 1.4.2)
       minitest (>= 5.1)
-      mutex_m
-      tzinfo (~> 2.0)
+      securerandom (>= 0.3)
+      tzinfo (~> 2.0, >= 2.0.5)
     after_do (0.4.0)
     appraisal (2.5.0)
       bundler
@@ -31,20 +32,21 @@ GEM
     base64 (0.2.0)
     bigdecimal (3.1.8)
     byebug (11.1.3)
-    concurrent-ruby (1.3.3)
+    concurrent-ruby (1.3.4)
     connection_pool (2.4.1)
     docile (1.4.1)
     drb (2.2.1)
     i18n (1.14.5)
       concurrent-ruby (~> 1.0)
-    minitest (5.24.1)
+    logger (1.6.0)
+    minitest (5.25.1)
     mocha (2.4.5)
       ruby2_keywords (>= 0.0.5)
-    mutex_m (0.2.0)
     mysql2 (0.5.6)
     rake (13.2.1)
     retriable (3.1.2)
     ruby2_keywords (0.0.5)
+    securerandom (0.3.1)
     simplecov (0.22.0)
       docile (~> 1.1)
       simplecov-html (~> 0.11)

data/gemfiles/activerecord_6.1.gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: ..
   specs:
-    lhm-shopify (4.3.0)
+    lhm-shopify (4.4.0)
       retriable (>= 3.0.0)
 
 GEM

data/gemfiles/activerecord_7.0.gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: ..
   specs:
-    lhm-shopify (4.3.0)
+    lhm-shopify (4.4.0)
       retriable (>= 3.0.0)
 
 GEM

data/gemfiles/activerecord_7.1.gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: ..
   specs:
-    lhm-shopify (4.3.0)
+    lhm-shopify (4.4.0)
       retriable (>= 3.0.0)
 
 GEM

data/gemfiles/activerecord_head.gemfile.lock

@@ -0,0 +1,88 @@
+GIT
+  remote: https://github.com/rails/rails.git
+  revision: f4a9b7618fc32f0d3b2c0ff03a3f34f4964cc553
+  branch: main
+  specs:
+    activemodel (8.0.0.alpha)
+      activesupport (= 8.0.0.alpha)
+    activerecord (8.0.0.alpha)
+      activemodel (= 8.0.0.alpha)
+      activesupport (= 8.0.0.alpha)
+      timeout (>= 0.4.0)
+    activesupport (8.0.0.alpha)
+      base64
+      bigdecimal
+      concurrent-ruby (~> 1.0, >= 1.3.1)
+      connection_pool (>= 2.2.5)
+      drb
+      i18n (>= 1.6, < 2)
+      logger (>= 1.4.2)
+      minitest (>= 5.1)
+      securerandom (>= 0.3)
+      tzinfo (~> 2.0, >= 2.0.5)
+
+PATH
+  remote: ..
+  specs:
+    lhm-shopify (4.4.0)
+      retriable (>= 3.0.0)
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    after_do (0.4.0)
+    appraisal (2.5.0)
+      bundler
+      rake
+      thor (>= 0.14.0)
+    base64 (0.2.0)
+    bigdecimal (3.1.8)
+    byebug (11.1.3)
+    concurrent-ruby (1.3.3)
+    connection_pool (2.4.1)
+    docile (1.4.1)
+    drb (2.2.1)
+    i18n (1.14.5)
+      concurrent-ruby (~> 1.0)
+    logger (1.6.0)
+    minitest (5.24.1)
+    mocha (2.4.5)
+      ruby2_keywords (>= 0.0.5)
+    mysql2 (0.5.6)
+    rake (13.2.1)
+    retriable (3.1.2)
+    ruby2_keywords (0.0.5)
+    securerandom (0.3.1)
+    simplecov (0.22.0)
+      docile (~> 1.1)
+      simplecov-html (~> 0.11)
+      simplecov_json_formatter (~> 0.1)
+    simplecov-html (0.12.3)
+    simplecov_json_formatter (0.1.4)
+    thor (1.3.1)
+    timeout (0.4.1)
+    toxiproxy (2.0.2)
+    trilogy (2.8.1)
+    tzinfo (2.0.6)
+      concurrent-ruby (~> 1.0)
+
+PLATFORMS
+  arm64-darwin-22
+  x86_64-linux
+
+DEPENDENCIES
+  activerecord!
+  after_do
+  appraisal
+  byebug
+  lhm-shopify!
+  minitest
+  mocha
+  mysql2
+  rake
+  simplecov
+  toxiproxy
+  trilogy
+
+BUNDLED WITH
+   2.2.22

data/lib/lhm/chunker.rb

@@ -37,6 +37,10 @@ module Lhm
       )
     end
 
+    def handle_max_binlog_exceeded_error
+      @throttler.backoff_stride
+    end
+
     def execute
       @start_time = Time.now
 
@@ -47,7 +51,18 @@ module Lhm
         top = upper_id(@next_to_insert, stride)
         verify_can_run
 
-        affected_rows = ChunkInsert.new(@migration, @connection, bottom, top, @retry_options).insert_and_return_count_of_rows_created
+        begin
+          affected_rows = ChunkInsert.new(@migration, @connection, bottom, top, @retry_options).insert_and_return_count_of_rows_created
+        rescue ActiveRecord::StatementInvalid => e
+          if e.message.downcase.include?("transaction required more than 'max_binlog_cache_size' bytes of storage")
+            Lhm.logger.info("Encountered max_binlog_cache_size error, attempting to reduce stride size")
+            handle_max_binlog_exceeded_error
+            next
+          else
+            raise e
+          end
+        end
+
         expected_rows = top - bottom + 1
 
         # Only log the chunker progress every 5 minutes instead of every iteration
@@ -110,6 +125,5 @@ module Lhm
       return if @chunk_finder.table_empty?
       @chunk_finder.validate
     end
-
   end
 end

data/lib/lhm/throttler/time.rb

@@ -5,6 +5,8 @@ module Lhm
 
       DEFAULT_TIMEOUT = 0.1
       DEFAULT_STRIDE = 2_000
+      DEFAULT_BACKOFF_REDUCTION_FACTOR = 0.2 # 20%
+      MIN_STRIDE_SIZE = 1
 
       attr_accessor :timeout_seconds
       attr_accessor :stride
@@ -12,6 +14,37 @@ module Lhm
       def initialize(options = {})
         @timeout_seconds = options[:delay] || DEFAULT_TIMEOUT
         @stride = options[:stride] || DEFAULT_STRIDE
+        @backoff_reduction_factor = options[:backoff_reduction_factor] || DEFAULT_BACKOFF_REDUCTION_FACTOR
+        @min_stride_size = options[:min_stride_size] || MIN_STRIDE_SIZE
+
+        if @backoff_reduction_factor >= 1 || @backoff_reduction_factor <= 0
+          raise ArgumentError, 'backoff_reduction_factor must be between greater than 0, and less than 1'
+        end
+
+        if @min_stride_size < 1
+          raise ArgumentError, 'min_stride_size must be an integer greater than 0'
+        end
+
+        if !@min_stride_size.is_a?(Integer)
+          raise ArgumentError, 'min_stride_size must be an integer'
+        end
+
+        if @min_stride_size > @stride
+          raise ArgumentError, 'min_stride_size must be less than or equal to stride'
+        end
+      end
+
+      def backoff_stride
+        new_stride = (@stride * (1 - @backoff_reduction_factor)).to_i
+
+        if new_stride == @stride
+          raise RuntimeError, "Cannot backoff any further"
+        end
+
+        if new_stride < @min_stride_size
+          raise RuntimeError, "Cannot reduce stride below #{@min_stride_size}"
+        end
+        @stride = new_stride
       end
 
       def execute

data/lib/lhm/version.rb

@@ -2,5 +2,5 @@
 # Schmidt
 
 module Lhm
-  VERSION = '4.3.0'
+  VERSION = '4.4.0'
 end

data/spec/integration/chunker_spec.rb

@@ -5,6 +5,7 @@ require File.expand_path(File.dirname(__FILE__)) + '/integration_helper'
 require 'lhm/table'
 require 'lhm/migration'
 
+
 describe Lhm::Chunker do
   include IntegrationHelper
 
@@ -17,6 +18,7 @@ describe Lhm::Chunker do
       @migration = Lhm::Migration.new(@origin, @destination)
       @logs = StringIO.new
       Lhm.logger = Logger.new(@logs)
+      set_max_binlog_size(1024 * 1024 * 1024) # necessary since some tests reduce binlog size (1gb default)
     end
 
     def log_messages
@@ -306,6 +308,70 @@ describe Lhm::Chunker do
         value(count_all(@destination.name)).must_equal(0)
       end
     end
+
+    it 'should reduce stride size if chunker runs into max_binlog_cache_size error' do
+      init_stride = 1000
+
+      # Create a bunch of users
+      n = 0
+      25.times do |i|
+        execute "BEGIN"
+        init_stride.times do # each batch is 10 * 1000 * i bytes, so each batch of 1000 will range from 10kb - 250kb
+          n += 1
+          id = n
+          username_data = "a" * 10 * i
+          execute "insert into origin (id, common) values (#{id}, '#{username_data}')"
+        end
+        execute "COMMIT"
+      end
+
+      # reduce binlog size to 8kb
+      set_max_binlog_size(1024 * 8)
+
+      throttler = Lhm::Throttler::Time.new(stride: init_stride )
+      chunker = Lhm::Chunker.new(
+        @migration, connection, { throttler: throttler  }
+      )
+
+      # start chunking
+      chunker.run
+      assert init_stride > throttler.stride
+    end
+
+    it 'should throw an error when stride cannot be reduced beyond min stride size' do
+      init_stride = 100
+      min_stride_size = 50
+
+      # Create a bunch of users
+      n = 0
+      25.times do |i|
+        execute "BEGIN"
+        init_stride.times do # each batch is init_stride * 250 bytes, so even at min_stride of 20,
+                             # batch_size will be greater than 4kb (50 * 250kb = 12.5kb)
+          n += 1
+          id = n
+          username_data = "a" * 250
+          execute "insert into origin (id, common) values (#{id}, '#{username_data}')"
+        end
+        execute "COMMIT"
+      end
+
+      # reduce binlog size to 4kb
+      set_max_binlog_size(1024 * 4)
+      throttler = Lhm::Throttler::Time.new(stride: init_stride, min_stride_size: min_stride_size, backoff_reduction_factor: 0.9)
+
+      chunker = Lhm::Chunker.new(
+        @migration, connection, { throttler: throttler  }
+      )
+
+      # start chunking
+      exception = assert_raises do
+        chunker.run
+      end
+
+      assert RuntimeError = exception.class
+      assert "Cannot reduce stride below #{min_stride_size}" == exception.message
+    end
   end
 
   def index_key(table_name, index_name)
@@ -315,4 +381,13 @@ describe Lhm::Chunker do
       index_name
     end
   end
+
+  def set_global_variable(name, value)
+    execute("set global #{name} = #{value}")
+    connection.reconnect!
+  end
+
+  def set_max_binlog_size(value)
+    set_global_variable('max_binlog_cache_size', value)
+  end
 end

data/spec/unit/throttler_spec.rb

@@ -111,6 +111,66 @@ describe Lhm::Throttler do
     end
   end
 
+  describe 'when using backoff functionality' do
+    it 'should backoff by default amount' do
+      @mock.setup_throttler(:time_throttler, stride: 100)
+      @mock.throttler.backoff_stride
+      value(@mock.throttler.stride).must_equal 80
+    end
+
+    it 'should backoff by specified amount' do
+      @mock.setup_throttler(:time_throttler, backoff_reduction_factor: 0.5, stride: 100)
+      @mock.throttler.backoff_stride
+      value(@mock.throttler.stride).must_equal 50
+    end
+
+    it 'should throw an error when backoff exceeds limit' do
+      @mock.setup_throttler(:time_throttler, backoff_reduction_factor: 0.2, stride: 1000, min_stride_size: 900)
+      proc { @mock.throttler.backoff_stride }.must_raise RuntimeError
+    end
+
+    it 'should throw an error when backoff cannot be done anymore' do
+      @mock.setup_throttler(:time_throttler, backoff_reduction_factor: 0.2, stride: 1, min_stride_size: 1)
+      proc { @mock.throttler.backoff_stride }.must_raise RuntimeError
+    end
+
+    it 'should throw an error when backoff reduction factor is not less than one' do
+      assert_raises ArgumentError do
+        @mock.setup_throttler(:time_throttler, backoff_reduction_factor: 1)
+      end
+    end
+
+    it 'should throw an error when backoff reduction factor is not greater than zero' do
+      assert_raises ArgumentError do
+        @mock.setup_throttler(:time_throttler, backoff_reduction_factor: 0)
+      end
+    end
+
+    it 'should throw an error when backoff reduction factor is negative' do
+      assert_raises ArgumentError do
+        @mock.setup_throttler(:time_throttler, backoff_reduction_factor: -0.5)
+      end
+    end
+
+    it 'should throw an error when min_stride_size is not an integer' do
+      assert_raises ArgumentError do
+        @mock.setup_throttler(:time_throttler, min_stride_size: 0.5)
+      end
+    end
+
+    it 'should throw an error when min_stride_size is not greater than 1' do
+      assert_raises ArgumentError do
+        @mock.setup_throttler(:time_throttler, min_stride_size: -12)
+      end
+    end
+
+    it 'should throw an error when min_stride_size is greater than inital stride size' do
+      assert_raises ArgumentError do
+        @mock.setup_throttler(:time_throttler, min_stride_size: 1000, stride: 500)
+      end
+    end
+  end
+
   describe '#throttler' do
 
     it 'returns the default Time based' do

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: lhm-shopify
 version: !ruby/object:Gem::Version
-  version: 4.3.0
+  version: 4.4.0
 platform: ruby
 authors:
 - SoundCloud
@@ -12,7 +12,7 @@ authors:
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2024-08-05 00:00:00.000000000 Z
+date: 2024-08-20 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: retriable
@@ -213,6 +213,7 @@ files:
 - gemfiles/activerecord_7.1.gemfile
 - gemfiles/activerecord_7.1.gemfile.lock
 - gemfiles/activerecord_head.gemfile
+- gemfiles/activerecord_head.gemfile.lock
 - lhm.gemspec
 - lib/lhm-shopify.rb
 - lib/lhm.rb
@@ -325,7 +326,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.5.16
+rubygems_version: 3.5.17
 signing_key:
 specification_version: 4
 summary: online schema changer for mysql