lhm-shopify 4.2.3 → 4.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 8e88f970c0a74075057e83cd9dddaff855c4b1260279604badc45af7d4f096f0
-  data.tar.gz: 722aefe84a0e97d6702b4b2706df2ad9c9084dec24be0e3e790fa4f4f535ea73
+  metadata.gz: 9b9ff81249ed6289557c131a6b6183dd77b494af67b71cb4845e69628092801e
+  data.tar.gz: ca4a71a59985f62685056833ad288a63ff33b0ca233a7a41571cb7f0963b096e
 SHA512:
-  metadata.gz: f423dbc69088a1e314e0b460a051f5a3438754382a5f26008e2ed1e754da771467a532c9f269cbfd2e1e0aa347e600b3dbabc436f8cc94c32d0d99819cb0f832
-  data.tar.gz: 047c7221c4eb28d5bc01a16287b5e813e46dbc6b7effae12ff6778660e8e295adb53490d9825c8ca714df6e00b3abef50cff522a2f29a0ae9c6834797e9c93e5
+  metadata.gz: ae9eedad45c7425785150927537a604084c60215c1bdfa796e7bc018fdddbaff2e431af97f5611d15ad5064ad9640f4af38c24f44712eb3567a66bba9f128038
+  data.tar.gz: 67135252c34441946ff069a1c6283545656a60b77306f8e6c019d319f0a164fd17dd898a21fcbfeba5490a52598812a804de52ad5fa498b6eee77a7731bc49ae

data/.github/workflows/test.yml

@@ -13,9 +13,10 @@ jobs:
 
     strategy:
       fail-fast: false
+      max-parallel: 8
       matrix:
-        activerecord: ["6.1", "7.0", "7.1"]
-        ruby: ["3.0", "3.1", "3.2", "3.3", "head"]
+        activerecord: ["6.1", "7.0", "7.1", "head"]
+        ruby: ["3.1", "3.2", "3.3", "head"]
         mysql: ["5.7", "8.0"]
         adapter: ["mysql2", "trilogy"]
         exclude:
@@ -23,6 +24,8 @@ jobs:
             ruby: head
           - activerecord: 7.0
             ruby: head
+          - activerecord: head
+            ruby: 3.0
 
     env:
       BUNDLE_GEMFILE: "${{ github.workspace }}/gemfiles/activerecord_${{ matrix.activerecord }}.gemfile"
@@ -40,7 +43,7 @@ jobs:
 
       - name: Setup MySQL and ProxySQL (docker-compose)
         # Might have to change to docker compose up -d (i.e. Compose V2) when the Ubuntu image changes the docker-compose version
-        run: docker-compose -f docker-compose-mysql-${{ matrix.mysql }}.yml up -d
+        run: docker compose -f docker-compose-mysql-${{ matrix.mysql }}.yml up -d
 
       - name: Wait until DBs are alive
         run: ./scripts/helpers/wait-for-dbs.sh

data/Appraisals

@@ -11,3 +11,7 @@ end
 appraise "activerecord-7.1" do
   gem "activerecord", "7.1.1"
 end
+
+appraise "activerecord-head" do
+  gem "activerecord", git: "https://github.com/rails/rails.git", branch: "main"
+end

data/CHANGELOG.md

@@ -1,5 +1,12 @@
 # Unreleased
 
+# 4.4.0 (Aug, 2024)
+* Add support for retrying chunks when running into max_binlog_cache_size exceeded error
+
+# 4.3.0 (Aug, 2024)
+* Drop support for Ruby 3.0, as it reached its EOL
+* Add support for next Rails version
+
 # 4.2.3 (Jul, 2024)
 * Fix check for warnings against PKs with line breaks
 

data/Gemfile.lock

@@ -1,61 +1,62 @@
 PATH
   remote: .
   specs:
-    lhm-shopify (4.2.3)
+    lhm-shopify (4.4.0)
       retriable (>= 3.0.0)
 
 GEM
   remote: https://rubygems.org/
   specs:
-    activemodel (7.1.1)
-      activesupport (= 7.1.1)
-    activerecord (7.1.1)
-      activemodel (= 7.1.1)
-      activesupport (= 7.1.1)
+    activemodel (7.2.0)
+      activesupport (= 7.2.0)
+    activerecord (7.2.0)
+      activemodel (= 7.2.0)
+      activesupport (= 7.2.0)
       timeout (>= 0.4.0)
-    activesupport (7.1.1)
+    activesupport (7.2.0)
       base64
       bigdecimal
-      concurrent-ruby (~> 1.0, >= 1.0.2)
+      concurrent-ruby (~> 1.0, >= 1.3.1)
       connection_pool (>= 2.2.5)
       drb
       i18n (>= 1.6, < 2)
+      logger (>= 1.4.2)
       minitest (>= 5.1)
-      mutex_m
-      tzinfo (~> 2.0)
+      securerandom (>= 0.3)
+      tzinfo (~> 2.0, >= 2.0.5)
     after_do (0.4.0)
     appraisal (2.5.0)
       bundler
       rake
       thor (>= 0.14.0)
-    base64 (0.1.1)
-    bigdecimal (3.1.4)
+    base64 (0.2.0)
+    bigdecimal (3.1.8)
     byebug (11.1.3)
-    concurrent-ruby (1.2.2)
+    concurrent-ruby (1.3.4)
     connection_pool (2.4.1)
-    docile (1.4.0)
-    drb (2.1.1)
-      ruby2_keywords
-    i18n (1.14.1)
+    docile (1.4.1)
+    drb (2.2.1)
+    i18n (1.14.5)
       concurrent-ruby (~> 1.0)
-    minitest (5.22.2)
-    mocha (2.1.0)
+    logger (1.6.0)
+    minitest (5.25.1)
+    mocha (2.4.5)
       ruby2_keywords (>= 0.0.5)
-    mutex_m (0.1.2)
-    mysql2 (0.5.5)
-    rake (13.0.6)
+    mysql2 (0.5.6)
+    rake (13.2.1)
     retriable (3.1.2)
     ruby2_keywords (0.0.5)
+    securerandom (0.3.1)
     simplecov (0.22.0)
       docile (~> 1.1)
       simplecov-html (~> 0.11)
       simplecov_json_formatter (~> 0.1)
     simplecov-html (0.12.3)
     simplecov_json_formatter (0.1.4)
-    thor (1.2.2)
-    timeout (0.4.0)
+    thor (1.3.1)
+    timeout (0.4.1)
     toxiproxy (2.0.2)
-    trilogy (2.6.0)
+    trilogy (2.8.1)
     tzinfo (2.0.6)
       concurrent-ruby (~> 1.0)
 

data/dev.yml

@@ -1,9 +1,7 @@
 name: lhm
 up:
-  - homebrew:
-      - mysql-client@5.7:
-          or: [mysql@5.7]
-          conflicts: [shopify/shopify/mysql-client, mysql-connector-c, mysql, mysql-client]
+  - packages:
+      - mysql-client@5.7
       - wget
   - ruby
   - bundler

data/gemfiles/activerecord_6.1.gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: ..
   specs:
-    lhm-shopify (4.2.3)
+    lhm-shopify (4.4.0)
       retriable (>= 3.0.0)
 
 GEM

data/gemfiles/activerecord_7.0.gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: ..
   specs:
-    lhm-shopify (4.2.3)
+    lhm-shopify (4.4.0)
       retriable (>= 3.0.0)
 
 GEM

data/gemfiles/activerecord_7.1.gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: ..
   specs:
-    lhm-shopify (4.2.3)
+    lhm-shopify (4.4.0)
       retriable (>= 3.0.0)
 
 GEM

data/gemfiles/activerecord_head.gemfile

@@ -0,0 +1,7 @@
+# This file was generated by Appraisal
+
+source "https://rubygems.org"
+
+gem "activerecord", git: "https://github.com/rails/rails.git", branch: "main"
+
+gemspec path: "../"

data/gemfiles/activerecord_head.gemfile.lock

@@ -0,0 +1,88 @@
+GIT
+  remote: https://github.com/rails/rails.git
+  revision: f4a9b7618fc32f0d3b2c0ff03a3f34f4964cc553
+  branch: main
+  specs:
+    activemodel (8.0.0.alpha)
+      activesupport (= 8.0.0.alpha)
+    activerecord (8.0.0.alpha)
+      activemodel (= 8.0.0.alpha)
+      activesupport (= 8.0.0.alpha)
+      timeout (>= 0.4.0)
+    activesupport (8.0.0.alpha)
+      base64
+      bigdecimal
+      concurrent-ruby (~> 1.0, >= 1.3.1)
+      connection_pool (>= 2.2.5)
+      drb
+      i18n (>= 1.6, < 2)
+      logger (>= 1.4.2)
+      minitest (>= 5.1)
+      securerandom (>= 0.3)
+      tzinfo (~> 2.0, >= 2.0.5)
+
+PATH
+  remote: ..
+  specs:
+    lhm-shopify (4.4.0)
+      retriable (>= 3.0.0)
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    after_do (0.4.0)
+    appraisal (2.5.0)
+      bundler
+      rake
+      thor (>= 0.14.0)
+    base64 (0.2.0)
+    bigdecimal (3.1.8)
+    byebug (11.1.3)
+    concurrent-ruby (1.3.3)
+    connection_pool (2.4.1)
+    docile (1.4.1)
+    drb (2.2.1)
+    i18n (1.14.5)
+      concurrent-ruby (~> 1.0)
+    logger (1.6.0)
+    minitest (5.24.1)
+    mocha (2.4.5)
+      ruby2_keywords (>= 0.0.5)
+    mysql2 (0.5.6)
+    rake (13.2.1)
+    retriable (3.1.2)
+    ruby2_keywords (0.0.5)
+    securerandom (0.3.1)
+    simplecov (0.22.0)
+      docile (~> 1.1)
+      simplecov-html (~> 0.11)
+      simplecov_json_formatter (~> 0.1)
+    simplecov-html (0.12.3)
+    simplecov_json_formatter (0.1.4)
+    thor (1.3.1)
+    timeout (0.4.1)
+    toxiproxy (2.0.2)
+    trilogy (2.8.1)
+    tzinfo (2.0.6)
+      concurrent-ruby (~> 1.0)
+
+PLATFORMS
+  arm64-darwin-22
+  x86_64-linux
+
+DEPENDENCIES
+  activerecord!
+  after_do
+  appraisal
+  byebug
+  lhm-shopify!
+  minitest
+  mocha
+  mysql2
+  rake
+  simplecov
+  toxiproxy
+  trilogy
+
+BUNDLED WITH
+   2.2.22

data/lib/lhm/chunker.rb

@@ -37,6 +37,10 @@ module Lhm
       )
     end
 
+    def handle_max_binlog_exceeded_error
+      @throttler.backoff_stride
+    end
+
     def execute
       @start_time = Time.now
 
@@ -47,7 +51,18 @@ module Lhm
         top = upper_id(@next_to_insert, stride)
         verify_can_run
 
-        affected_rows = ChunkInsert.new(@migration, @connection, bottom, top, @retry_options).insert_and_return_count_of_rows_created
+        begin
+          affected_rows = ChunkInsert.new(@migration, @connection, bottom, top, @retry_options).insert_and_return_count_of_rows_created
+        rescue ActiveRecord::StatementInvalid => e
+          if e.message.downcase.include?("transaction required more than 'max_binlog_cache_size' bytes of storage")
+            Lhm.logger.info("Encountered max_binlog_cache_size error, attempting to reduce stride size")
+            handle_max_binlog_exceeded_error
+            next
+          else
+            raise e
+          end
+        end
+
         expected_rows = top - bottom + 1
 
         # Only log the chunker progress every 5 minutes instead of every iteration
@@ -110,6 +125,5 @@ module Lhm
       return if @chunk_finder.table_empty?
       @chunk_finder.validate
     end
-
   end
 end

data/lib/lhm/sql_helper.rb

@@ -65,7 +65,7 @@ module Lhm
     end
 
     def struct_key(struct, key)
-      keys = if struct.is_a? Hash
+      keys = if struct.respond_to?(:keys)
                struct.keys
              else
                struct.members

data/lib/lhm/throttler/time.rb

@@ -5,6 +5,8 @@ module Lhm
 
       DEFAULT_TIMEOUT = 0.1
       DEFAULT_STRIDE = 2_000
+      DEFAULT_BACKOFF_REDUCTION_FACTOR = 0.2 # 20%
+      MIN_STRIDE_SIZE = 1
 
       attr_accessor :timeout_seconds
       attr_accessor :stride
@@ -12,6 +14,37 @@ module Lhm
       def initialize(options = {})
         @timeout_seconds = options[:delay] || DEFAULT_TIMEOUT
         @stride = options[:stride] || DEFAULT_STRIDE
+        @backoff_reduction_factor = options[:backoff_reduction_factor] || DEFAULT_BACKOFF_REDUCTION_FACTOR
+        @min_stride_size = options[:min_stride_size] || MIN_STRIDE_SIZE
+
+        if @backoff_reduction_factor >= 1 || @backoff_reduction_factor <= 0
+          raise ArgumentError, 'backoff_reduction_factor must be between greater than 0, and less than 1'
+        end
+
+        if @min_stride_size < 1
+          raise ArgumentError, 'min_stride_size must be an integer greater than 0'
+        end
+
+        if !@min_stride_size.is_a?(Integer)
+          raise ArgumentError, 'min_stride_size must be an integer'
+        end
+
+        if @min_stride_size > @stride
+          raise ArgumentError, 'min_stride_size must be less than or equal to stride'
+        end
+      end
+
+      def backoff_stride
+        new_stride = (@stride * (1 - @backoff_reduction_factor)).to_i
+
+        if new_stride == @stride
+          raise RuntimeError, "Cannot backoff any further"
+        end
+
+        if new_stride < @min_stride_size
+          raise RuntimeError, "Cannot reduce stride below #{@min_stride_size}"
+        end
+        @stride = new_stride
       end
 
       def execute

data/lib/lhm/version.rb

@@ -2,5 +2,5 @@
 # Schmidt
 
 module Lhm
-  VERSION = '4.2.3'
+  VERSION = '4.4.0'
 end

data/spec/integration/chunker_spec.rb

@@ -5,6 +5,7 @@ require File.expand_path(File.dirname(__FILE__)) + '/integration_helper'
 require 'lhm/table'
 require 'lhm/migration'
 
+
 describe Lhm::Chunker do
   include IntegrationHelper
 
@@ -17,6 +18,7 @@ describe Lhm::Chunker do
       @migration = Lhm::Migration.new(@origin, @destination)
       @logs = StringIO.new
       Lhm.logger = Logger.new(@logs)
+      set_max_binlog_size(1024 * 1024 * 1024) # necessary since some tests reduce binlog size (1gb default)
     end
 
     def log_messages
@@ -306,6 +308,70 @@ describe Lhm::Chunker do
         value(count_all(@destination.name)).must_equal(0)
       end
     end
+
+    it 'should reduce stride size if chunker runs into max_binlog_cache_size error' do
+      init_stride = 1000
+
+      # Create a bunch of users
+      n = 0
+      25.times do |i|
+        execute "BEGIN"
+        init_stride.times do # each batch is 10 * 1000 * i bytes, so each batch of 1000 will range from 10kb - 250kb
+          n += 1
+          id = n
+          username_data = "a" * 10 * i
+          execute "insert into origin (id, common) values (#{id}, '#{username_data}')"
+        end
+        execute "COMMIT"
+      end
+
+      # reduce binlog size to 8kb
+      set_max_binlog_size(1024 * 8)
+
+      throttler = Lhm::Throttler::Time.new(stride: init_stride )
+      chunker = Lhm::Chunker.new(
+        @migration, connection, { throttler: throttler  }
+      )
+
+      # start chunking
+      chunker.run
+      assert init_stride > throttler.stride
+    end
+
+    it 'should throw an error when stride cannot be reduced beyond min stride size' do
+      init_stride = 100
+      min_stride_size = 50
+
+      # Create a bunch of users
+      n = 0
+      25.times do |i|
+        execute "BEGIN"
+        init_stride.times do # each batch is init_stride * 250 bytes, so even at min_stride of 20,
+                             # batch_size will be greater than 4kb (50 * 250kb = 12.5kb)
+          n += 1
+          id = n
+          username_data = "a" * 250
+          execute "insert into origin (id, common) values (#{id}, '#{username_data}')"
+        end
+        execute "COMMIT"
+      end
+
+      # reduce binlog size to 4kb
+      set_max_binlog_size(1024 * 4)
+      throttler = Lhm::Throttler::Time.new(stride: init_stride, min_stride_size: min_stride_size, backoff_reduction_factor: 0.9)
+
+      chunker = Lhm::Chunker.new(
+        @migration, connection, { throttler: throttler  }
+      )
+
+      # start chunking
+      exception = assert_raises do
+        chunker.run
+      end
+
+      assert RuntimeError = exception.class
+      assert "Cannot reduce stride below #{min_stride_size}" == exception.message
+    end
   end
 
   def index_key(table_name, index_name)
@@ -315,4 +381,13 @@ describe Lhm::Chunker do
       index_name
     end
   end
+
+  def set_global_variable(name, value)
+    execute("set global #{name} = #{value}")
+    connection.reconnect!
+  end
+
+  def set_max_binlog_size(value)
+    set_global_variable('max_binlog_cache_size', value)
+  end
 end

data/spec/integration/lhm_spec.rb

@@ -328,8 +328,7 @@ describe Lhm do
           :collate => collation,
         })
 
-        result = select_one('SELECT login from users')
-        result = result['login'] if result.respond_to?(:has_key?)
+        result = select_value('SELECT login from users')
         value(result).must_equal('a user')
       end
     end
@@ -353,8 +352,7 @@ describe Lhm do
           :collate => collation,
         })
 
-        result = select_one('SELECT `fnord` from users')
-        result = result['fnord'] if result.respond_to?(:has_key?)
+        result = select_value('SELECT `fnord` from users')
         value(result).must_equal('Superfriends')
       end
     end
@@ -380,8 +378,7 @@ describe Lhm do
              :collate => 'utf8mb4_unicode_ci',
            })
 
-        result = select_one('SELECT `user_name` from users')
-        result = result['user_name'] if result.respond_to?(:has_key?)
+        result = select_value('SELECT `user_name` from users')
         value(result).must_equal('a user')
       end
     end
@@ -410,8 +407,7 @@ describe Lhm do
            :collate => nil,
          })
 
-        result = select_one('SELECT `ref` from users')
-        result = result['ref'] if result.respond_to?(:has_key?)
+        result = select_value('SELECT `ref` from users')
         value(result).must_equal(10)
       end
     end
@@ -437,8 +433,7 @@ describe Lhm do
            :collate => collation,
          })
 
-        result = select_one('SELECT `fnord` from users')
-        result = result['fnord'] if result.respond_to?(:has_key?)
+        result = select_value('SELECT `fnord` from users')
         assert_nil(result)
       end
     end
@@ -462,8 +457,7 @@ describe Lhm do
           :collate => collation,
         })
 
-        result = select_one('SELECT `user_name` from users')
-        result = result['user_name'] if result.respond_to?(:has_key?)
+        result = select_value('SELECT `user_name` from users')
         value(result).must_equal('a user')
       end
     end
@@ -489,8 +483,7 @@ describe Lhm do
           :collate => collation,
         })
 
-        result = select_one('SELECT `user_name` from users')
-        result = result['user_name'] if result.respond_to?(:has_key?)
+        result = select_value('SELECT `user_name` from users')
         value(result).must_equal('a user')
       end
     end

data/spec/unit/throttler_spec.rb

@@ -111,6 +111,66 @@ describe Lhm::Throttler do
     end
   end
 
+  describe 'when using backoff functionality' do
+    it 'should backoff by default amount' do
+      @mock.setup_throttler(:time_throttler, stride: 100)
+      @mock.throttler.backoff_stride
+      value(@mock.throttler.stride).must_equal 80
+    end
+
+    it 'should backoff by specified amount' do
+      @mock.setup_throttler(:time_throttler, backoff_reduction_factor: 0.5, stride: 100)
+      @mock.throttler.backoff_stride
+      value(@mock.throttler.stride).must_equal 50
+    end
+
+    it 'should throw an error when backoff exceeds limit' do
+      @mock.setup_throttler(:time_throttler, backoff_reduction_factor: 0.2, stride: 1000, min_stride_size: 900)
+      proc { @mock.throttler.backoff_stride }.must_raise RuntimeError
+    end
+
+    it 'should throw an error when backoff cannot be done anymore' do
+      @mock.setup_throttler(:time_throttler, backoff_reduction_factor: 0.2, stride: 1, min_stride_size: 1)
+      proc { @mock.throttler.backoff_stride }.must_raise RuntimeError
+    end
+
+    it 'should throw an error when backoff reduction factor is not less than one' do
+      assert_raises ArgumentError do
+        @mock.setup_throttler(:time_throttler, backoff_reduction_factor: 1)
+      end
+    end
+
+    it 'should throw an error when backoff reduction factor is not greater than zero' do
+      assert_raises ArgumentError do
+        @mock.setup_throttler(:time_throttler, backoff_reduction_factor: 0)
+      end
+    end
+
+    it 'should throw an error when backoff reduction factor is negative' do
+      assert_raises ArgumentError do
+        @mock.setup_throttler(:time_throttler, backoff_reduction_factor: -0.5)
+      end
+    end
+
+    it 'should throw an error when min_stride_size is not an integer' do
+      assert_raises ArgumentError do
+        @mock.setup_throttler(:time_throttler, min_stride_size: 0.5)
+      end
+    end
+
+    it 'should throw an error when min_stride_size is not greater than 1' do
+      assert_raises ArgumentError do
+        @mock.setup_throttler(:time_throttler, min_stride_size: -12)
+      end
+    end
+
+    it 'should throw an error when min_stride_size is greater than inital stride size' do
+      assert_raises ArgumentError do
+        @mock.setup_throttler(:time_throttler, min_stride_size: 1000, stride: 500)
+      end
+    end
+  end
+
   describe '#throttler' do
 
     it 'returns the default Time based' do

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: lhm-shopify
 version: !ruby/object:Gem::Version
-  version: 4.2.3
+  version: 4.4.0
 platform: ruby
 authors:
 - SoundCloud
@@ -12,7 +12,7 @@ authors:
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2024-07-23 00:00:00.000000000 Z
+date: 2024-08-20 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: retriable
@@ -212,6 +212,8 @@ files:
 - gemfiles/activerecord_7.0.gemfile.lock
 - gemfiles/activerecord_7.1.gemfile
 - gemfiles/activerecord_7.1.gemfile.lock
+- gemfiles/activerecord_head.gemfile
+- gemfiles/activerecord_head.gemfile.lock
 - lhm.gemspec
 - lib/lhm-shopify.rb
 - lib/lhm.rb
@@ -324,7 +326,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.5.16
+rubygems_version: 3.5.17
 signing_key:
 specification_version: 4
 summary: online schema changer for mysql