lhm-shopify 4.4.0 → 4.4.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 9b9ff81249ed6289557c131a6b6183dd77b494af67b71cb4845e69628092801e
-  data.tar.gz: ca4a71a59985f62685056833ad288a63ff33b0ca233a7a41571cb7f0963b096e
+  metadata.gz: 938d28fc99ea36454247de34a03ff3a202f0892e11f590b0c41556de4e041ee4
+  data.tar.gz: e28790317276015c4ffaeb9150c7e2a5eeb3a3fad1713604bda670225c6e181b
 SHA512:
-  metadata.gz: ae9eedad45c7425785150927537a604084c60215c1bdfa796e7bc018fdddbaff2e431af97f5611d15ad5064ad9640f4af38c24f44712eb3567a66bba9f128038
-  data.tar.gz: 67135252c34441946ff069a1c6283545656a60b77306f8e6c019d319f0a164fd17dd898a21fcbfeba5490a52598812a804de52ad5fa498b6eee77a7731bc49ae
+  metadata.gz: 6eeca296f8626d383bccc0978440f26dd05c7a4da002648101db578e553d4a407da4238e35cf7ee18a21e821d7c1f66a30c55bd665dff9863bdad80e3f6c60b9
+  data.tar.gz: 78b1799fd87375c9042b3340b40db4439702b007977632afe104800bd01b12aad34e44f9694b3d58e9a277a3a42a8785794d87c31d2b56c3a9b59f448b60eda4

data/CHANGELOG.md

@@ -1,5 +1,11 @@
 # Unreleased
 
+# 4.4.2 (Sep, 2024)
+* Allow caller to set the algorithm that will be used for DDL ALTER TABLE operations
+
+# 4.4.1 (Aug, 2024)
+* Extend max_binlog_cache_size exceeded error handling to all throttlers
+
 # 4.4.0 (Aug, 2024)
 * Add support for retrying chunks when running into max_binlog_cache_size exceeded error
 

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    lhm-shopify (4.4.0)
+    lhm-shopify (4.4.2)
       retriable (>= 3.0.0)
 
 GEM

data/README.md

@@ -207,6 +207,35 @@ Or to set that as default throttler, use the following (for instance in a Rails
 Lhm.setup_throttler(:threads_running_throttler)
 ```
 
+### Retrying chunks using Throttler 
+
+If chunks fail due to the MySQL `max_binlog_cache_size` being exceeded, the Chunker class will call the `backoff_stride` method on the throttler to reduce the stride size and retry the chunk. The default backoff factor is 0.2 (reducing the chunk size by 20% with each failure), and the minimum stride size is 1 (the stride can be reduced until each chunk is 1 row). Default values can be overridden (see below). To disable backoff, set `min_stride_size` equal to `stride_size`; this will throw an exception when `max_binlog_cache_size` is exceeded. Configuration options should be passed into the time throttler constructor or in the `throttler_options` of change_table when using a time throttler.
+
+```ruby
+## Configure on throttler instance 
+my_throttler = Lhm::Throttler::ThreadsRunning.new(stride: 2000, delay: 1, backoff_reduction_factor: 0.1, min_stride_size: 10)
+
+Lhm.change_table :users, throttler: my_throttler  do |m|
+  ...
+end
+
+## Or pass as throttler options 
+Lhm.change_table :users, {
+  throttler: :time_throttler,
+  throttler_options: {
+    stride: 2000,
+    delay: 1,
+    backoff_reduction_factor: 0.1,
+    min_stride_size: 10
+  }
+} do |t|
+  ...
+end
+```
+
+### Custom Throttlers with retry functionality 
+If using your own throttler, ensure that the class definition includes the `Lhm::Throttler::BackoffReduction` to retry failed chunks.  
+
 ## Table rename strategies
 
 There are two different table rename strategies available: `LockedSwitcher` and
@@ -311,7 +340,7 @@ open coverage/index.html
 ```
 
 ### Merging for a new version
-When creating a PR for a new version, make sure that th version has been bumped in `lib/lhm/version.rb`. Then run the following code snippet to ensure the everything is consistent, otherwise
+When creating a PR for a new version, make sure that the version has been bumped in `lib/lhm/version.rb`. Then run the following code snippet to ensure the everything is consistent, otherwise
 the gem will not publish.
 ```bash
 bundle install

data/gemfiles/activerecord_6.1.gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: ..
   specs:
-    lhm-shopify (4.4.0)
+    lhm-shopify (4.4.2)
       retriable (>= 3.0.0)
 
 GEM

data/gemfiles/activerecord_7.0.gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: ..
   specs:
-    lhm-shopify (4.4.0)
+    lhm-shopify (4.4.2)
       retriable (>= 3.0.0)
 
 GEM

data/gemfiles/activerecord_7.1.gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: ..
   specs:
-    lhm-shopify (4.4.0)
+    lhm-shopify (4.4.2)
       retriable (>= 3.0.0)
 
 GEM

data/gemfiles/activerecord_head.gemfile.lock

@@ -24,7 +24,7 @@ GIT
 PATH
   remote: ..
   specs:
-    lhm-shopify (4.4.0)
+    lhm-shopify (4.4.2)
       retriable (>= 3.0.0)
 
 GEM

data/lib/lhm/chunker.rb

@@ -37,10 +37,6 @@ module Lhm
       )
     end
 
-    def handle_max_binlog_exceeded_error
-      @throttler.backoff_stride
-    end
-
     def execute
       @start_time = Time.now
 
@@ -54,9 +50,9 @@ module Lhm
         begin
           affected_rows = ChunkInsert.new(@migration, @connection, bottom, top, @retry_options).insert_and_return_count_of_rows_created
         rescue ActiveRecord::StatementInvalid => e
-          if e.message.downcase.include?("transaction required more than 'max_binlog_cache_size' bytes of storage")
+          if e.message.downcase.include?("transaction required more than 'max_binlog_cache_size' bytes of storage") && @throttler.respond_to?(:backoff_stride)
             Lhm.logger.info("Encountered max_binlog_cache_size error, attempting to reduce stride size")
-            handle_max_binlog_exceeded_error
+            @throttler.backoff_stride
             next
           else
             raise e

data/lib/lhm/migrator.rb

@@ -32,14 +32,16 @@ module Lhm
     #   end
     #
     # @param [String] statement SQL alter statement
+    # @param [String] algorithm Algorithm that will be used in the DDL operation
     # @note
     #
     #   Don't write the table name directly into the statement. Use the #name
     #   getter instead, because the alter statement will be executed against a
     #   temporary table.
     #
-    def ddl(statement)
-      statements << statement
+    def ddl(statement, algorithm: nil)
+      full_statement = algorithm ? "#{statement}, ALGORITHM=#{algorithm}" : statement
+      statements << full_statement
     end
 
     # Add a column to a table
@@ -52,8 +54,9 @@ module Lhm
     #
     # @param [String] name Name of the column to add
     # @param [String] definition Valid SQL column definition
-    def add_column(name, definition)
-      ddl('alter table `%s` add column `%s` %s, ALGORITHM=INPLACE' % [@name, name, definition])
+    # @param [String] algorithm Algorithm that will be used in the DDL operation
+    def add_column(name, definition, algorithm: 'INPLACE')
+      ddl('alter table `%s` add column `%s` %s' % [@name, name, definition], algorithm:)
     end
 
     # Change an existing column to a new definition
@@ -84,7 +87,8 @@ module Lhm
     #
     # @param [String] old Name of the column to change
     # @param [String] nu New name to use for the column
-    def rename_column(old, nu)
+    # @param [String] algorithm Algorithm that will be used in the DDL operation
+    def rename_column(old, nu, algorithm: 'INPLACE')
       col = @origin.columns[old.to_s]
 
       definition = col[:type]
@@ -94,7 +98,7 @@ module Lhm
       definition += " COMMENT #{@connection.quote(col[:comment])}" if col[:comment]
       definition += " COLLATE #{@connection.quote(col[:collate])}" if col[:collate]
 
-      ddl('alter table `%s` change column `%s` `%s` %s, ALGORITHM=INPLACE' % [@name, old, nu, definition])
+      ddl('alter table `%s` change column `%s` `%s` %s' % [@name, old, nu, definition], algorithm:)
       @renames[old.to_s] = nu.to_s
     end
 
@@ -107,8 +111,9 @@ module Lhm
     #   end
     #
     # @param [String] name Name of the column to delete
-    def remove_column(name)
-      ddl('alter table `%s` drop `%s`, ALGORITHM=INPLACE' % [@name, name])
+    # @param [String] algorithm Algorithm that will be used in the DDL operation
+    def remove_column(name, algorithm: 'INPLACE')
+      ddl('alter table `%s` drop `%s`' % [@name, name], algorithm:)
     end
 
     # Add an index to a table

data/lib/lhm/throttler/backoff_reduction.rb

@@ -0,0 +1,42 @@
+module Lhm
+  module Throttler
+    module BackoffReduction
+      DEFAULT_BACKOFF_REDUCTION_FACTOR = 0.2
+      MIN_STRIDE_SIZE = 1
+
+      def initialize(options = {})
+        @backoff_reduction_factor = options[:backoff_reduction_factor] || DEFAULT_BACKOFF_REDUCTION_FACTOR
+        @min_stride_size = options[:min_stride_size] || MIN_STRIDE_SIZE
+
+        if @backoff_reduction_factor >= 1 || @backoff_reduction_factor <= 0
+          raise ArgumentError, 'backoff_reduction_factor must be between greater than 0, and less than 1'
+        end
+
+        if @min_stride_size < 1
+          raise ArgumentError, 'min_stride_size must be an integer greater than 0'
+        end
+
+        if !@min_stride_size.is_a?(Integer)
+          raise ArgumentError, 'min_stride_size must be an integer'
+        end
+
+        if @min_stride_size > @stride
+          raise ArgumentError, 'min_stride_size must be less than or equal to stride'
+        end
+      end
+
+      def backoff_stride
+        new_stride = (@stride * (1 - @backoff_reduction_factor)).to_i
+
+        if new_stride == @stride
+          raise RuntimeError, "Cannot backoff any further"
+        end
+
+        if new_stride < @min_stride_size
+          raise RuntimeError, "Cannot reduce stride below #{@min_stride_size}"
+        end
+        @stride = new_stride
+      end
+    end
+  end
+end

data/lib/lhm/throttler/replica_lag.rb

@@ -13,6 +13,7 @@ module Lhm
 
     class ReplicaLag
       include Command
+      include BackoffReduction
 
       INITIAL_TIMEOUT = 0.1
       DEFAULT_STRIDE = 2_000
@@ -29,6 +30,8 @@ module Lhm
         @replicas = {}
         @get_config = options[:current_config]
         @check_only = options[:check_only]
+
+        super
       end
 
       def execute

data/lib/lhm/throttler/threads_running.rb

@@ -2,6 +2,7 @@ module Lhm
   module Throttler
     class ThreadsRunning
       include Command
+      include BackoffReduction
 
       DEFAULT_STRIDE = 2_000
       DEFAULT_INITIAL_TIMEOUT = 0.1
@@ -17,6 +18,8 @@ module Lhm
         @timeout_seconds = @initial_timeout_seconds
         @healthy_range = options[:healthy_range] || DEFAULT_HEALTHY_RANGE
         @connection = options[:connection]
+
+        super
       end
 
       def threads_running

data/lib/lhm/throttler/time.rb

@@ -2,6 +2,7 @@ module Lhm
   module Throttler
     class Time
       include Command
+      include BackoffReduction
 
       DEFAULT_TIMEOUT = 0.1
       DEFAULT_STRIDE = 2_000
@@ -14,37 +15,8 @@ module Lhm
       def initialize(options = {})
         @timeout_seconds = options[:delay] || DEFAULT_TIMEOUT
         @stride = options[:stride] || DEFAULT_STRIDE
-        @backoff_reduction_factor = options[:backoff_reduction_factor] || DEFAULT_BACKOFF_REDUCTION_FACTOR
-        @min_stride_size = options[:min_stride_size] || MIN_STRIDE_SIZE
 
-        if @backoff_reduction_factor >= 1 || @backoff_reduction_factor <= 0
-          raise ArgumentError, 'backoff_reduction_factor must be between greater than 0, and less than 1'
-        end
-
-        if @min_stride_size < 1
-          raise ArgumentError, 'min_stride_size must be an integer greater than 0'
-        end
-
-        if !@min_stride_size.is_a?(Integer)
-          raise ArgumentError, 'min_stride_size must be an integer'
-        end
-
-        if @min_stride_size > @stride
-          raise ArgumentError, 'min_stride_size must be less than or equal to stride'
-        end
-      end
-
-      def backoff_stride
-        new_stride = (@stride * (1 - @backoff_reduction_factor)).to_i
-
-        if new_stride == @stride
-          raise RuntimeError, "Cannot backoff any further"
-        end
-
-        if new_stride < @min_stride_size
-          raise RuntimeError, "Cannot reduce stride below #{@min_stride_size}"
-        end
-        @stride = new_stride
+        super
       end
 
       def execute

data/lib/lhm/throttler.rb

@@ -1,3 +1,4 @@
+require 'lhm/throttler/backoff_reduction'
 require 'lhm/throttler/time'
 require 'lhm/throttler/replica_lag'
 require 'lhm/throttler/slave_lag'

data/lib/lhm/version.rb

@@ -2,5 +2,5 @@
 # Schmidt
 
 module Lhm
-  VERSION = '4.4.0'
+  VERSION = '4.4.2'
 end

data/spec/unit/migrator_spec.rb

@@ -101,6 +101,14 @@ describe Lhm::Migrator do
       ])
     end
 
+    it 'should add a column using the passed algorithm' do
+      @creator.add_column('logins', 'INT(12)', algorithm: 'COPY')
+
+      value(@creator.statements).must_equal([
+        'alter table `lhmn_alt` add column `logins` INT(12), ALGORITHM=COPY'
+      ])
+    end
+
     it 'should remove a column' do
       @creator.remove_column('logins')
 
@@ -109,6 +117,14 @@ describe Lhm::Migrator do
       ])
     end
 
+    it 'should remove a column using the passed algorithm' do
+      @creator.remove_column('logins', algorithm: 'COPY')
+
+      value(@creator.statements).must_equal([
+        'alter table `lhmn_alt` drop `logins`, ALGORITHM=COPY'
+      ])
+    end
+
     it 'should change a column' do
       @creator.change_column('logins', 'INT(11)')
 
@@ -157,4 +173,18 @@ describe Lhm::Migrator do
         .must_equal('alter table `lhmn_alt` add column `last` VARCHAR(64), ALGORITHM=INPLACE')
     end
   end
+
+  describe 'multiple changes using the passed algorithm' do
+    it 'should add two columns' do
+      @creator.add_column('first', 'VARCHAR(64)', algorithm: 'COPY')
+      @creator.add_column('last', 'VARCHAR(64)', algorithm: 'COPY')
+      value(@creator.statements.length).must_equal(2)
+
+      value(@creator.statements[0])
+        .must_equal('alter table `lhmn_alt` add column `first` VARCHAR(64), ALGORITHM=COPY')
+
+      value(@creator.statements[1])
+        .must_equal('alter table `lhmn_alt` add column `last` VARCHAR(64), ALGORITHM=COPY')
+    end
+  end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: lhm-shopify
 version: !ruby/object:Gem::Version
-  version: 4.4.0
+  version: 4.4.2
 platform: ruby
 authors:
 - SoundCloud
@@ -12,7 +12,7 @@ authors:
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2024-08-20 00:00:00.000000000 Z
+date: 2024-09-09 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: retriable
@@ -239,6 +239,7 @@ files:
 - lib/lhm/table_name.rb
 - lib/lhm/test_support.rb
 - lib/lhm/throttler.rb
+- lib/lhm/throttler/backoff_reduction.rb
 - lib/lhm/throttler/replica_lag.rb
 - lib/lhm/throttler/slave_lag.rb
 - lib/lhm/throttler/threads_running.rb
@@ -326,7 +327,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.5.17
+rubygems_version: 3.5.18
 signing_key:
 specification_version: 4
 summary: online schema changer for mysql