lhm-shopify 4.4.1 → 4.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 0ef875eccb1a986bb82ef5ee9d981e229ca54fc92e58f61b6e48768928e48baf
-  data.tar.gz: d206155bcf16a94c2dc78622826c35b57f7e09b4d9fff1ddad9726d1a48d61fa
+  metadata.gz: 56380d3c695085c3fdc5ea625014ad7a90948a0a227d9f543cfc23efc8ec9403
+  data.tar.gz: b06d43bb5c314e15846ee46ed50acfe8a076069558b0fd8f94a620a574bd2a38
 SHA512:
-  metadata.gz: 6f3c37c8387638aebcdcb3caced88a39b998247a55a130fc8eade81b4c37622af8acdb570e234ce1c3fa25562ec460844e9cc48e9d9d5d1fe4284c9a6d7f4be8
-  data.tar.gz: d5a1b197b4260aa494a30de528cdf6e5cfa6e8b81c9f40b40eb9e71fcdcae1c73db6f955d599d5930d581b7c1be7623dc0f96e0112073aa063f178157b1b52c6
+  metadata.gz: 0244a6df042163ed02c77dd37bfaa800ff99a8a5af518561694fb071eb630dafd73cc12146be38a41e1e02d1f5523f7ae9ebcbd90a9696ae267d2b1e5e425957
+  data.tar.gz: 4fda241d6c7f516d96c679c1fee6bfdabc5c13e8e27baa95623406f0fb568294271fac81a836e3611b4f9d3243ef14fa7d433693603b3ae5056fdd7a31acc66c

data/.github/workflows/test.yml

@@ -15,17 +15,19 @@ jobs:
       fail-fast: false
       max-parallel: 8
       matrix:
-        activerecord: ["6.1", "7.0", "7.1", "head"]
+        activerecord: ["7.2", "8.0", "head"]
         ruby: ["3.1", "3.2", "3.3", "head"]
         mysql: ["5.7", "8.0"]
         adapter: ["mysql2", "trilogy"]
         exclude:
           - activerecord: 6.1
             ruby: head
-          - activerecord: 7.0
+          - activerecord: 7.2
             ruby: head
+          - activerecord: 8.0
+            ruby: 3.1
           - activerecord: head
-            ruby: 3.0
+            ruby: 3.1
 
     env:
       BUNDLE_GEMFILE: "${{ github.workspace }}/gemfiles/activerecord_${{ matrix.activerecord }}.gemfile"

data/.ruby-version

@@ -1 +1 @@
-3.2.2
+3.3.6

data/Appraisals

@@ -1,15 +1,9 @@
-appraise "activerecord-6.1" do
-  gem "activerecord", "6.1.0"
-  gem "activerecord-trilogy-adapter"
+appraise "activerecord-7.2" do
+  gem "activerecord", "7.2.2.1"
 end
 
-appraise "activerecord-7.0" do
-  gem "activerecord", "7.0.8"
-  gem "activerecord-trilogy-adapter"
-end
-
-appraise "activerecord-7.1" do
-  gem "activerecord", "7.1.1"
+appraise "activerecord-8.0" do
+  gem "activerecord", "8.0.1"
 end
 
 appraise "activerecord-head" do

data/CHANGELOG.md

@@ -1,5 +1,15 @@
 # Unreleased
 
+# 4.5.0 (Feb, 2025)
+* Update test matrix to include Rails 8
+* Add option to force tables to use default engine
+* Change default algorithm for add/rename/remove column operations
+* Drop support for ActiveRecord version 6.1, as it has reached EOL
+* Add support for tables with generated columns
+
+# 4.4.2 (Sep, 2024)
+* Allow caller to set the algorithm that will be used for DDL ALTER TABLE operations
+
 # 4.4.1 (Aug, 2024)
 * Extend max_binlog_cache_size exceeded error handling to all throttlers
 

data/Gemfile.lock

@@ -1,20 +1,21 @@
 PATH
   remote: .
   specs:
-    lhm-shopify (4.4.1)
+    lhm-shopify (4.5.0)
       retriable (>= 3.0.0)
 
 GEM
   remote: https://rubygems.org/
   specs:
-    activemodel (7.2.0)
-      activesupport (= 7.2.0)
-    activerecord (7.2.0)
-      activemodel (= 7.2.0)
-      activesupport (= 7.2.0)
+    activemodel (8.0.1)
+      activesupport (= 8.0.1)
+    activerecord (8.0.1)
+      activemodel (= 8.0.1)
+      activesupport (= 8.0.1)
       timeout (>= 0.4.0)
-    activesupport (7.2.0)
+    activesupport (8.0.1)
       base64
+      benchmark (>= 0.3)
       bigdecimal
       concurrent-ruby (~> 1.0, >= 1.3.1)
       connection_pool (>= 2.2.5)
@@ -24,45 +25,50 @@ GEM
       minitest (>= 5.1)
       securerandom (>= 0.3)
       tzinfo (~> 2.0, >= 2.0.5)
+      uri (>= 0.13.1)
     after_do (0.4.0)
     appraisal (2.5.0)
       bundler
       rake
       thor (>= 0.14.0)
     base64 (0.2.0)
-    bigdecimal (3.1.8)
+    benchmark (0.4.0)
+    bigdecimal (3.1.9)
     byebug (11.1.3)
-    concurrent-ruby (1.3.4)
-    connection_pool (2.4.1)
+    concurrent-ruby (1.3.5)
+    connection_pool (2.5.0)
     docile (1.4.1)
     drb (2.2.1)
-    i18n (1.14.5)
+    i18n (1.14.7)
       concurrent-ruby (~> 1.0)
-    logger (1.6.0)
-    minitest (5.25.1)
+    logger (1.6.5)
+    minitest (5.25.4)
     mocha (2.4.5)
       ruby2_keywords (>= 0.0.5)
     mysql2 (0.5.6)
+    ostruct (0.6.1)
     rake (13.2.1)
     retriable (3.1.2)
     ruby2_keywords (0.0.5)
-    securerandom (0.3.1)
+    securerandom (0.4.1)
     simplecov (0.22.0)
       docile (~> 1.1)
       simplecov-html (~> 0.11)
       simplecov_json_formatter (~> 0.1)
     simplecov-html (0.12.3)
     simplecov_json_formatter (0.1.4)
-    thor (1.3.1)
-    timeout (0.4.1)
+    thor (1.3.2)
+    timeout (0.4.3)
     toxiproxy (2.0.2)
     trilogy (2.8.1)
     tzinfo (2.0.6)
       concurrent-ruby (~> 1.0)
+    uri (1.0.2)
 
 PLATFORMS
   arm64-darwin-21
   arm64-darwin-22
+  arm64-darwin-23
   x86_64-darwin-20
   x86_64-linux
 
@@ -72,13 +78,15 @@ DEPENDENCIES
   appraisal
   byebug
   lhm-shopify!
+  logger
   minitest
   mocha
   mysql2
+  ostruct
   rake
   simplecov
   toxiproxy
   trilogy
 
 BUNDLED WITH
-   2.2.22
+   2.6.3

data/README.md

@@ -207,6 +207,35 @@ Or to set that as default throttler, use the following (for instance in a Rails
 Lhm.setup_throttler(:threads_running_throttler)
 ```
 
+### Retrying chunks using Throttler 
+
+If chunks fail due to the MySQL `max_binlog_cache_size` being exceeded, the Chunker class will call the `backoff_stride` method on the throttler to reduce the stride size and retry the chunk. The default backoff factor is 0.2 (reducing the chunk size by 20% with each failure), and the minimum stride size is 1 (the stride can be reduced until each chunk is 1 row). Default values can be overridden (see below). To disable backoff, set `min_stride_size` equal to `stride_size`; this will throw an exception when `max_binlog_cache_size` is exceeded. Configuration options should be passed into the time throttler constructor or in the `throttler_options` of change_table when using a time throttler.
+
+```ruby
+## Configure on throttler instance 
+my_throttler = Lhm::Throttler::ThreadsRunning.new(stride: 2000, delay: 1, backoff_reduction_factor: 0.1, min_stride_size: 10)
+
+Lhm.change_table :users, throttler: my_throttler  do |m|
+  ...
+end
+
+## Or pass as throttler options 
+Lhm.change_table :users, {
+  throttler: :time_throttler,
+  throttler_options: {
+    stride: 2000,
+    delay: 1,
+    backoff_reduction_factor: 0.1,
+    min_stride_size: 10
+  }
+} do |t|
+  ...
+end
+```
+
+### Custom Throttlers with retry functionality 
+If using your own throttler, ensure that the class definition includes the `Lhm::Throttler::BackoffReduction` to retry failed chunks.  
+
 ## Table rename strategies
 
 There are two different table rename strategies available: `LockedSwitcher` and
@@ -311,7 +340,7 @@ open coverage/index.html
 ```
 
 ### Merging for a new version
-When creating a PR for a new version, make sure that th version has been bumped in `lib/lhm/version.rb`. Then run the following code snippet to ensure the everything is consistent, otherwise
+When creating a PR for a new version, make sure that the version has been bumped in `lib/lhm/version.rb`. Then run the following code snippet to ensure the everything is consistent, otherwise
 the gem will not publish.
 ```bash
 bundle install

data/gemfiles/{activerecord_6.1.gemfile → activerecord_7.2.gemfile}

@@ -2,7 +2,6 @@
 
 source "https://rubygems.org"
 
-gem "activerecord", "6.1.0"
-gem "activerecord-trilogy-adapter"
+gem "activerecord", "7.2.2.1"
 
 gemspec path: "../"

data/gemfiles/{activerecord_6.1.gemfile.lock → activerecord_7.2.gemfile.lock}

@@ -1,76 +1,90 @@
 PATH
   remote: ..
   specs:
-    lhm-shopify (4.4.1)
+    lhm-shopify (4.5.0)
       retriable (>= 3.0.0)
 
 GEM
   remote: https://rubygems.org/
   specs:
-    activemodel (6.1.0)
-      activesupport (= 6.1.0)
-    activerecord (6.1.0)
-      activemodel (= 6.1.0)
-      activesupport (= 6.1.0)
-    activerecord-trilogy-adapter (3.1.2)
-      activerecord (>= 6.0.a, < 7.1.a)
-      trilogy (>= 2.4.0)
-    activesupport (6.1.0)
-      concurrent-ruby (~> 1.0, >= 1.0.2)
+    activemodel (7.2.2.1)
+      activesupport (= 7.2.2.1)
+    activerecord (7.2.2.1)
+      activemodel (= 7.2.2.1)
+      activesupport (= 7.2.2.1)
+      timeout (>= 0.4.0)
+    activesupport (7.2.2.1)
+      base64
+      benchmark (>= 0.3)
+      bigdecimal
+      concurrent-ruby (~> 1.0, >= 1.3.1)
+      connection_pool (>= 2.2.5)
+      drb
       i18n (>= 1.6, < 2)
+      logger (>= 1.4.2)
       minitest (>= 5.1)
-      tzinfo (~> 2.0)
-      zeitwerk (~> 2.3)
+      securerandom (>= 0.3)
+      tzinfo (~> 2.0, >= 2.0.5)
     after_do (0.4.0)
     appraisal (2.5.0)
       bundler
       rake
       thor (>= 0.14.0)
+    base64 (0.2.0)
+    benchmark (0.4.0)
+    bigdecimal (3.1.9)
     byebug (11.1.3)
-    concurrent-ruby (1.2.2)
+    concurrent-ruby (1.3.5)
+    connection_pool (2.5.0)
     docile (1.4.0)
-    i18n (1.14.1)
+    drb (2.2.1)
+    i18n (1.14.7)
       concurrent-ruby (~> 1.0)
-    minitest (5.22.2)
+    logger (1.6.5)
+    minitest (5.25.4)
     mocha (2.1.0)
       ruby2_keywords (>= 0.0.5)
     mysql2 (0.5.5)
+    ostruct (0.6.1)
     rake (13.0.6)
     retriable (3.1.2)
     ruby2_keywords (0.0.5)
+    securerandom (0.4.1)
     simplecov (0.22.0)
       docile (~> 1.1)
       simplecov-html (~> 0.11)
       simplecov_json_formatter (~> 0.1)
     simplecov-html (0.12.3)
     simplecov_json_formatter (0.1.4)
-    thor (1.2.2)
+    thor (1.3.2)
+    timeout (0.4.3)
     toxiproxy (2.0.2)
-    trilogy (2.6.0)
+    trilogy (2.9.0)
     tzinfo (2.0.6)
       concurrent-ruby (~> 1.0)
-    zeitwerk (2.6.12)
 
 PLATFORMS
   arm64-darwin-21
   arm64-darwin-22
+  arm64-darwin-23
   x86_64-darwin-20
   x86_64-linux
 
 DEPENDENCIES
-  activerecord (= 6.1.0)
-  activerecord-trilogy-adapter
+  activerecord (= 7.2.2.1)
   after_do
   appraisal
   byebug
   lhm-shopify!
+  logger
   minitest
   mocha
   mysql2
+  ostruct
   rake
   simplecov
   toxiproxy
   trilogy
 
 BUNDLED WITH
-   2.2.22
+   2.6.3

data/gemfiles/{activerecord_7.1.gemfile → activerecord_8.0.gemfile}

@@ -2,6 +2,6 @@
 
 source "https://rubygems.org"
 
-gem "activerecord", "7.1.1"
+gem "activerecord", "8.0.1"
 
 gemspec path: "../"

data/gemfiles/{activerecord_7.1.gemfile.lock → activerecord_8.0.gemfile.lock}

@@ -1,83 +1,92 @@
 PATH
   remote: ..
   specs:
-    lhm-shopify (4.4.1)
+    lhm-shopify (4.5.0)
       retriable (>= 3.0.0)
 
 GEM
   remote: https://rubygems.org/
   specs:
-    activemodel (7.1.1)
-      activesupport (= 7.1.1)
-    activerecord (7.1.1)
-      activemodel (= 7.1.1)
-      activesupport (= 7.1.1)
+    activemodel (8.0.1)
+      activesupport (= 8.0.1)
+    activerecord (8.0.1)
+      activemodel (= 8.0.1)
+      activesupport (= 8.0.1)
       timeout (>= 0.4.0)
-    activesupport (7.1.1)
+    activesupport (8.0.1)
       base64
+      benchmark (>= 0.3)
       bigdecimal
-      concurrent-ruby (~> 1.0, >= 1.0.2)
+      concurrent-ruby (~> 1.0, >= 1.3.1)
       connection_pool (>= 2.2.5)
       drb
       i18n (>= 1.6, < 2)
+      logger (>= 1.4.2)
       minitest (>= 5.1)
-      mutex_m
-      tzinfo (~> 2.0)
+      securerandom (>= 0.3)
+      tzinfo (~> 2.0, >= 2.0.5)
+      uri (>= 0.13.1)
     after_do (0.4.0)
     appraisal (2.5.0)
       bundler
       rake
       thor (>= 0.14.0)
-    base64 (0.1.1)
-    bigdecimal (3.1.4)
+    base64 (0.2.0)
+    benchmark (0.4.0)
+    bigdecimal (3.1.9)
     byebug (11.1.3)
-    concurrent-ruby (1.2.2)
-    connection_pool (2.4.1)
+    concurrent-ruby (1.3.5)
+    connection_pool (2.5.0)
     docile (1.4.0)
-    drb (2.1.1)
-      ruby2_keywords
+    drb (2.2.1)
     i18n (1.14.1)
       concurrent-ruby (~> 1.0)
+    logger (1.6.1)
     minitest (5.22.2)
     mocha (2.1.0)
       ruby2_keywords (>= 0.0.5)
-    mutex_m (0.1.2)
     mysql2 (0.5.5)
+    ostruct (0.6.1)
     rake (13.0.6)
     retriable (3.1.2)
     ruby2_keywords (0.0.5)
+    securerandom (0.4.1)
     simplecov (0.22.0)
       docile (~> 1.1)
       simplecov-html (~> 0.11)
       simplecov_json_formatter (~> 0.1)
     simplecov-html (0.12.3)
     simplecov_json_formatter (0.1.4)
-    thor (1.2.2)
-    timeout (0.4.0)
+    thor (1.3.2)
+    timeout (0.4.3)
     toxiproxy (2.0.2)
-    trilogy (2.6.0)
+    trilogy (2.9.0)
     tzinfo (2.0.6)
       concurrent-ruby (~> 1.0)
+    uri (1.0.2)
 
 PLATFORMS
   arm64-darwin-21
   arm64-darwin-22
+  arm64-darwin-23
   x86_64-darwin-20
   x86_64-linux
 
 DEPENDENCIES
-  activerecord (= 7.1.1)
+  activerecord (= 8.0.1)
   after_do
   appraisal
   byebug
   lhm-shopify!
+  logger
   minitest
   mocha
   mysql2
+  ostruct
   rake
   simplecov
   toxiproxy
   trilogy
 
 BUNDLED WITH
-   2.2.22
+   2.6.3

data/gemfiles/activerecord_head.gemfile.lock

@@ -1,16 +1,17 @@
 GIT
   remote: https://github.com/rails/rails.git
-  revision: f4a9b7618fc32f0d3b2c0ff03a3f34f4964cc553
+  revision: bdad2707835949a96dd29e747b8c324593efd4d8
   branch: main
   specs:
-    activemodel (8.0.0.alpha)
-      activesupport (= 8.0.0.alpha)
-    activerecord (8.0.0.alpha)
-      activemodel (= 8.0.0.alpha)
-      activesupport (= 8.0.0.alpha)
+    activemodel (8.1.0.alpha)
+      activesupport (= 8.1.0.alpha)
+    activerecord (8.1.0.alpha)
+      activemodel (= 8.1.0.alpha)
+      activesupport (= 8.1.0.alpha)
       timeout (>= 0.4.0)
-    activesupport (8.0.0.alpha)
+    activesupport (8.1.0.alpha)
       base64
+      benchmark (>= 0.3)
       bigdecimal
       concurrent-ruby (~> 1.0, >= 1.3.1)
       connection_pool (>= 2.2.5)
@@ -20,11 +21,12 @@ GIT
       minitest (>= 5.1)
       securerandom (>= 0.3)
       tzinfo (~> 2.0, >= 2.0.5)
+      uri (>= 0.13.1)
 
 PATH
   remote: ..
   specs:
-    lhm-shopify (4.4.1)
+    lhm-shopify (4.5.0)
       retriable (>= 3.0.0)
 
 GEM
@@ -36,38 +38,42 @@ GEM
       rake
       thor (>= 0.14.0)
     base64 (0.2.0)
-    bigdecimal (3.1.8)
+    benchmark (0.4.0)
+    bigdecimal (3.1.9)
     byebug (11.1.3)
-    concurrent-ruby (1.3.3)
-    connection_pool (2.4.1)
+    concurrent-ruby (1.3.5)
+    connection_pool (2.5.0)
     docile (1.4.1)
     drb (2.2.1)
-    i18n (1.14.5)
+    i18n (1.14.7)
       concurrent-ruby (~> 1.0)
-    logger (1.6.0)
-    minitest (5.24.1)
+    logger (1.6.5)
+    minitest (5.25.4)
     mocha (2.4.5)
       ruby2_keywords (>= 0.0.5)
     mysql2 (0.5.6)
+    ostruct (0.6.1)
     rake (13.2.1)
     retriable (3.1.2)
     ruby2_keywords (0.0.5)
-    securerandom (0.3.1)
+    securerandom (0.4.1)
     simplecov (0.22.0)
       docile (~> 1.1)
       simplecov-html (~> 0.11)
       simplecov_json_formatter (~> 0.1)
     simplecov-html (0.12.3)
     simplecov_json_formatter (0.1.4)
-    thor (1.3.1)
-    timeout (0.4.1)
+    thor (1.3.2)
+    timeout (0.4.3)
     toxiproxy (2.0.2)
-    trilogy (2.8.1)
+    trilogy (2.9.0)
     tzinfo (2.0.6)
       concurrent-ruby (~> 1.0)
+    uri (1.0.2)
 
 PLATFORMS
   arm64-darwin-22
+  arm64-darwin-23
   x86_64-linux
 
 DEPENDENCIES
@@ -76,13 +82,15 @@ DEPENDENCIES
   appraisal
   byebug
   lhm-shopify!
+  logger
   minitest
   mocha
   mysql2
+  ostruct
   rake
   simplecov
   toxiproxy
   trilogy
 
 BUNDLED WITH
-   2.2.22
+   2.6.3

data/lhm.gemspec

@@ -36,4 +36,6 @@ Gem::Specification.new do |s|
   s.add_development_dependency 'toxiproxy'
   s.add_development_dependency 'appraisal'
   s.add_development_dependency 'byebug'
+  s.add_development_dependency 'ostruct'
+  s.add_development_dependency 'logger'
 end

data/lib/lhm/intersection.rb

@@ -4,10 +4,11 @@
 module Lhm
   #  Determine and format columns common to origin and destination.
   class Intersection
-    def initialize(origin, destination, renames = {})
+    def initialize(origin, destination, renames = {}, generated_column_names = [])
       @origin = origin
       @destination = destination
       @renames = renames
+      @generated_column_names = generated_column_names
     end
 
     def origin
@@ -21,7 +22,7 @@ module Lhm
     private
 
     def common
-      (@origin.columns.keys & @destination.columns.keys).sort
+      ((@origin.columns.keys & @destination.columns.keys) - @generated_column_names).sort
     end
 
     module Joiners

data/lib/lhm/invoker.rb

@@ -21,9 +21,11 @@ module Lhm
 
     attr_reader :migrator, :connection
 
-    def initialize(origin, connection)
+    def initialize(origin, connection, options = {})
       @connection = connection
-      @migrator = Migrator.new(origin, connection)
+      @migrator = Migrator.new(origin, connection, options)
+      @options = options
+      normalize_options(@options)
     end
 
     def set_session_lock_wait_timeouts
@@ -45,17 +47,16 @@ module Lhm
       end
     end
 
-    def run(options = {})
-      normalize_options(options)
+    def run
       set_session_lock_wait_timeouts
       migration = @migrator.run
       entangler = Entangler.new(migration, @connection)
 
       entangler.run do
-        options[:verifier] ||= Proc.new { |conn| triggers_still_exist?(conn, entangler) }
-        Chunker.new(migration, @connection, options).run
+        @options[:verifier] ||= Proc.new { |conn| triggers_still_exist?(conn, entangler) }
+        Chunker.new(migration, @connection, @options).run
         raise "Required triggers do not exist" unless triggers_still_exist?(@connection, entangler)
-        if options[:atomic_switch]
+        if @options[:atomic_switch]
           AtomicSwitcher.new(migration, @connection).run
         else
           LockedSwitcher.new(migration, @connection).run

data/lib/lhm/migration.rb

@@ -8,12 +8,13 @@ module Lhm
   class Migration
     attr_reader :origin, :destination, :conditions, :renames
 
-    def initialize(origin, destination, conditions = nil, renames = {}, time = Time.now)
+    def initialize(origin, destination, conditions = nil, renames = {}, time = Time.now, generated_column_names = [])
       @origin = origin
       @destination = destination
       @conditions = conditions
       @renames = renames
       @table_name = TableName.new(@origin.name, time)
+      @generated_column_names = generated_column_names
     end
 
     def archive_name
@@ -21,7 +22,7 @@ module Lhm
     end
 
     def intersection
-      Intersection.new(@origin, @destination, @renames)
+      Intersection.new(@origin, @destination, @renames, @generated_column_names)
     end
 
     def origin_name

data/lib/lhm/migrator.rb

@@ -15,12 +15,13 @@ module Lhm
 
     attr_reader :name, :statements, :connection, :conditions, :renames, :origin
 
-    def initialize(table, connection = nil)
+    def initialize(table, connection = nil, options = {})
       @connection = connection
       @origin = table
       @name = table.destination_name
       @statements = []
       @renames = {}
+      @options = options
     end
 
     # Alter a table with a custom statement
@@ -32,14 +33,16 @@ module Lhm
     #   end
     #
     # @param [String] statement SQL alter statement
+    # @param [String] algorithm Algorithm that will be used in the DDL operation
     # @note
     #
     #   Don't write the table name directly into the statement. Use the #name
     #   getter instead, because the alter statement will be executed against a
     #   temporary table.
     #
-    def ddl(statement)
-      statements << statement
+    def ddl(statement, algorithm: nil)
+      full_statement = algorithm ? "#{statement}, ALGORITHM=#{algorithm}" : statement
+      statements << full_statement
     end
 
     # Add a column to a table
@@ -52,8 +55,9 @@ module Lhm
     #
     # @param [String] name Name of the column to add
     # @param [String] definition Valid SQL column definition
-    def add_column(name, definition)
-      ddl('alter table `%s` add column `%s` %s, ALGORITHM=INPLACE' % [@name, name, definition])
+    # @param [String] algorithm Algorithm that will be used in the DDL operation
+    def add_column(name, definition, algorithm: 'COPY')
+      ddl('alter table `%s` add column `%s` %s' % [@name, name, definition], algorithm:)
     end
 
     # Change an existing column to a new definition
@@ -84,7 +88,8 @@ module Lhm
     #
     # @param [String] old Name of the column to change
     # @param [String] nu New name to use for the column
-    def rename_column(old, nu)
+    # @param [String] algorithm Algorithm that will be used in the DDL operation
+    def rename_column(old, nu, algorithm: 'COPY')
       col = @origin.columns[old.to_s]
 
       definition = col[:type]
@@ -94,7 +99,7 @@ module Lhm
       definition += " COMMENT #{@connection.quote(col[:comment])}" if col[:comment]
       definition += " COLLATE #{@connection.quote(col[:collate])}" if col[:collate]
 
-      ddl('alter table `%s` change column `%s` `%s` %s, ALGORITHM=INPLACE' % [@name, old, nu, definition])
+      ddl('alter table `%s` change column `%s` `%s` %s' % [@name, old, nu, definition], algorithm:)
       @renames[old.to_s] = nu.to_s
     end
 
@@ -107,8 +112,9 @@ module Lhm
     #   end
     #
     # @param [String] name Name of the column to delete
-    def remove_column(name)
-      ddl('alter table `%s` drop `%s`, ALGORITHM=INPLACE' % [@name, name])
+    # @param [String] algorithm Algorithm that will be used in the DDL operation
+    def remove_column(name, algorithm: 'COPY')
+      ddl('alter table `%s` drop `%s`' % [@name, name], algorithm:)
     end
 
     # Add an index to a table
@@ -210,13 +216,22 @@ module Lhm
       @statements.each do |stmt|
         @connection.execute(tagged(stmt))
       end
-      Migration.new(@origin, destination_read, conditions, renames)
+      Migration.new(@origin, destination_read, conditions, renames, Time.now, generated_column_names)
+    end
+
+    def generated_column_names
+      @connection.columns(@origin.name).select(&:virtual?).map(&:name)
     end
 
     def destination_create
       original    = %{CREATE TABLE `#{ @origin.name }`}
       replacement = %{CREATE TABLE `#{ @origin.destination_name }`}
       stmt = @origin.ddl.gsub(original, replacement)
+
+      if @options[:force_default_engine]
+        stmt = stmt.sub(/ENGINE=\w+\s/, "")
+      end
+
       @connection.execute(tagged(stmt))
 
       Lhm.logger.info("Created destination table #{@origin.destination_name}")

data/lib/lhm/sql_retry.rb

@@ -199,6 +199,7 @@ module Lhm
           /The MySQL server is running with the --read-only option so it cannot execute this statement/,
         ],
         Trilogy::ConnectionError => nil,
+        Trilogy::TimeoutError => nil,
       }
 
       if ActiveRecord::VERSION::STRING >= "7.1"

data/lib/lhm/throttler/replica_lag.rb

@@ -134,7 +134,7 @@ module Lhm
             raise ArgumentError, "Expected #{config_proc.inspect} to respond to `call`"
           end
         else
-          db_config
+          ActiveRecord::Base.connection_pool.db_config.configuration_hash.dup
         end
         config.deep_symbolize_keys!
         config[:host] = @host
@@ -147,23 +147,6 @@ module Lhm
         Lhm.logger.info "Unable to connect and/or query #{host}: #{e}"
         [nil]
       end
-
-      private
-
-      def db_config
-        if ar_supports_db_config?
-          ActiveRecord::Base.connection_pool.db_config.configuration_hash.dup
-        else
-          ActiveRecord::Base.connection_pool.spec.config.dup
-        end
-      end
-
-      def ar_supports_db_config?
-        # https://api.rubyonrails.org/v6.0/classes/ActiveRecord/ConnectionAdapters/ConnectionPool.html <-- has spec
-        # vs
-        # https://api.rubyonrails.org/v6.1/classes/ActiveRecord/ConnectionAdapters/ConnectionPool.html <-- has db_config
-        ActiveRecord::VERSION::MAJOR > 6 || ActiveRecord::VERSION::MAJOR == 6 && ActiveRecord::VERSION::MINOR >= 1
-      end
     end
   end
 end

data/lib/lhm/version.rb

@@ -2,5 +2,5 @@
 # Schmidt
 
 module Lhm
-  VERSION = '4.4.1'
+  VERSION = '4.5.0'
 end

data/lib/lhm.rb

@@ -54,9 +54,9 @@ module Lhm
   def change_table(table_name, options = {}, &block)
     with_flags(options) do
       origin = Table.parse(table_name, connection)
-      invoker = Invoker.new(origin, connection)
+      invoker = Invoker.new(origin, connection, options)
       block.call(invoker.migrator)
-      invoker.run(options)
+      invoker.run
       true
     end
   end

data/spec/fixtures/myisam_users.ddl

@@ -0,0 +1,10 @@
+CREATE TABLE `myisam_users` (
+  `id` int(11) NOT NULL AUTO_INCREMENT,
+  `reference` int(11) DEFAULT NULL,
+  `username` varchar(255) DEFAULT NULL,
+  `group` varchar(255) DEFAULT 'Superfriends',
+  `created_at` datetime DEFAULT NULL,
+  `comment` varchar(20) DEFAULT NULL,
+  `description` text,
+  PRIMARY KEY (`id`)
+) ENGINE=MyISAM CHARSET=utf8

data/spec/integration/lhm_spec.rb

@@ -531,6 +531,61 @@ describe Lhm do
       end
     end
 
+    it 'works when table has generated columns' do
+      table_create(:users)
+      execute("insert into `users` set id = 1, `username` = 'memyself'")
+      execute("insert into `users` set id = 2, `username` = 'youyourself'")
+
+      # Add a generated column
+      Lhm.change_table(:users) do |t|
+        t.add_column(:sample_generated_column, 'VARCHAR(255) GENERATED ALWAYS AS (SUBSTRING(`username`, -2))')
+      end
+
+      # Without the handling of generated columns
+      Lhm::Migrator.any_instance.stubs(:generated_column_names).returns([])
+      # without the Migration passing in generated columns to Intersection, we observe an error as an attempt to write
+      # directly into generated columns will fail.
+      exception = assert_raises ActiveRecord::StatementInvalid do
+        Lhm.change_table(:users) do |t|
+          t.add_column(:sample_additional_column, "VARCHAR(255)")
+        end
+      end
+      assert_match "The value specified for generated column 'sample_generated_column' in table 'lhmn_users' is not allowed.", exception.message
+
+      Lhm.cleanup(true)
+
+      # With the handling of generated columns
+      Lhm::Migrator.any_instance.unstub(:generated_column_names)
+      # As we are now skipping the writing to generated columns, this migration should succeed
+      Lhm.change_table(:users) do |t|
+        t.add_column(:sample_additional_column, "VARCHAR(255)")
+      end
+
+      replica do
+        # new column is added
+        value(table_read(:users).columns['sample_additional_column']).must_equal({
+          :type => 'varchar(255)',
+          :is_nullable => 'YES',
+          :column_default => nil,
+          :comment => '',
+          :collate => collation,
+        })
+
+        # generated column remains intact
+        value(table_read(:users).columns['sample_generated_column']).must_equal({
+          :type => 'varchar(255)',
+          :is_nullable => 'YES',
+          :column_default => nil,
+          :comment => '',
+          :collate => collation,
+        })
+      end
+
+      result = select_one('SELECT sample_generated_column FROM users')
+      # generated column populated appropriately
+      assert_match "lf", result["sample_generated_column"]
+    end
+
     it 'works when mysql reserved words are used' do
       table_create(:lines)
       execute("insert into `lines` set id = 1, `between` = 'foo'")
@@ -553,6 +608,39 @@ describe Lhm do
       end
     end
 
+    it 'creates the shadow table with the default engine when the `force_default_engine` option is used' do
+      table_create(:myisam_users)
+
+      engine = select_value("SELECT ENGINE FROM information_schema.TABLES WHERE TABLE_NAME = 'myisam_users'")
+      value(engine).must_equal("MyISAM")
+
+      Lhm.change_table(:myisam_users) do |t|
+        t.add_column(:logins, "INT(12) DEFAULT '0'", algorithm: "COPY")
+      end
+
+      engine = select_value("SELECT ENGINE FROM information_schema.TABLES WHERE TABLE_NAME = 'myisam_users'")
+      value(engine).must_equal("MyISAM")
+
+      Lhm.change_table(:myisam_users, force_default_engine: true) do |t|
+        t.remove_column(:logins)
+      end
+
+      engine = select_value("SELECT ENGINE FROM information_schema.TABLES WHERE TABLE_NAME = 'myisam_users'")
+      value(engine).must_equal("InnoDB")
+    end
+
+    it "should not fail using the default algorithms when changing tables with fulltext indexes" do
+      table_create(:users)
+      execute("DROP INDEX `index_with_a_custom_name` ON `users`")
+      execute("CREATE FULLTEXT INDEX `index_with_a_custom_name` ON `users` (`username`, `group`)")
+
+      Lhm.change_table(:users) do |t|
+        t.add_column(:email, "VARCHAR(255)")
+      end
+
+      value(table_read(:users).columns).must_include("email")
+    end
+
     describe 'parallel' do
       it 'should perserve inserts during migration' do
         50.times { |n| execute("insert into users set reference = '#{ n }'") }
@@ -616,7 +704,13 @@ describe Lhm do
         table_create(:users)
         100.times { |n| execute("insert into users set reference = '#{ n }'") }
 
-        assert_raises ActiveRecord::StatementInvalid do
+        error = if ActiveRecord.version >= ::Gem::Version.new('8.1.0.alpha')
+          ActiveRecord::ConnectionNotEstablished
+        else
+          ActiveRecord::StatementInvalid
+        end
+
+        assert_raises error do
           Toxiproxy[:mysql_master].down do
             Lhm.change_table(:users, :atomic_switch => false) do |t|
               t.ddl("ALTER TABLE #{t.name} CHANGE id id bigint (20) NOT NULL")

data/spec/integration/sql_retry/db_connection_helper.rb

@@ -20,7 +20,7 @@ class DBConnectionHelper
         :database => test_db_name,
         :port => db_config[key]['port']
       )
-      conn = conn.connection
+      conn = conn.respond_to?(:lease_connection) ? conn.lease_connection : conn.connection
       init_with_dummy_data(conn) if with_data
       conn
     end

data/spec/integration/sql_retry/lock_wait_spec.rb

@@ -81,10 +81,12 @@ describe Lhm::SqlRetry do
     logs = @logger.string.split("\n")
     assert_equal 2, logs.length
 
-    assert logs.first.include?("Lock wait timeout exceeded; try restarting transaction' - 1 tries")
+    assert logs.first.include?("Lock wait timeout exceeded; try restarting transaction")
+    assert logs.first.include?("- 1 tries")
     assert logs.first.include?("0.2 seconds until the next try")
 
-    assert logs.last.include?("Lock wait timeout exceeded; try restarting transaction' - 2 tries")
+    assert logs.last.include?("Lock wait timeout exceeded; try restarting transaction")
+    assert logs.last.include?("- 2 tries")
     assert logs.last.include?("0.2 seconds until the next try")
   end
 

data/spec/unit/migrator_spec.rb

@@ -97,7 +97,15 @@ describe Lhm::Migrator do
       @creator.add_column('logins', 'INT(12)')
 
       value(@creator.statements).must_equal([
-        'alter table `lhmn_alt` add column `logins` INT(12), ALGORITHM=INPLACE'
+        'alter table `lhmn_alt` add column `logins` INT(12), ALGORITHM=COPY'
+      ])
+    end
+
+    it 'should add a column using the passed algorithm' do
+      @creator.add_column('logins', 'INT(12)', algorithm: 'COPY')
+
+      value(@creator.statements).must_equal([
+        'alter table `lhmn_alt` add column `logins` INT(12), ALGORITHM=COPY'
       ])
     end
 
@@ -105,7 +113,15 @@ describe Lhm::Migrator do
       @creator.remove_column('logins')
 
       value(@creator.statements).must_equal([
-        'alter table `lhmn_alt` drop `logins`, ALGORITHM=INPLACE'
+        'alter table `lhmn_alt` drop `logins`, ALGORITHM=COPY'
+      ])
+    end
+
+    it 'should remove a column using the passed algorithm' do
+      @creator.remove_column('logins', algorithm: 'COPY')
+
+      value(@creator.statements).must_equal([
+        'alter table `lhmn_alt` drop `logins`, ALGORITHM=COPY'
       ])
     end
 
@@ -150,6 +166,20 @@ describe Lhm::Migrator do
       @creator.add_column('last', 'VARCHAR(64)')
       value(@creator.statements.length).must_equal(2)
 
+      value(@creator.statements[0])
+        .must_equal('alter table `lhmn_alt` add column `first` VARCHAR(64), ALGORITHM=COPY')
+
+      value(@creator.statements[1])
+        .must_equal('alter table `lhmn_alt` add column `last` VARCHAR(64), ALGORITHM=COPY')
+    end
+  end
+
+  describe 'multiple changes using the passed algorithm' do
+    it 'should add two columns' do
+      @creator.add_column('first', 'VARCHAR(64)', algorithm: 'INPLACE')
+      @creator.add_column('last', 'VARCHAR(64)', algorithm: 'INPLACE')
+      value(@creator.statements.length).must_equal(2)
+
       value(@creator.statements[0])
         .must_equal('alter table `lhmn_alt` add column `first` VARCHAR(64), ALGORITHM=INPLACE')
 

data/spec/unit/throttler/replica_lag_spec.rb

@@ -66,11 +66,7 @@ describe Lhm::Throttler::Replica do
     describe 'with active record config' do
       it 'logs and creates client' do
         active_record_config = { username: 'user', password: 'pw', database: 'db' }
-        if ActiveRecord::VERSION::MAJOR > 6 || ActiveRecord::VERSION::MAJOR == 6 && ActiveRecord::VERSION::MINOR >= 1
-          ActiveRecord::Base.stubs(:connection_pool).returns(stub(db_config: stub(configuration_hash: active_record_config)))
-        else
-          ActiveRecord::Base.stubs(:connection_pool).returns(stub(spec: stub(config: active_record_config)))
-        end
+        ActiveRecord::Base.stubs(:connection_pool).returns(stub(db_config: stub(configuration_hash: active_record_config)))
 
         DATABASE.client.stubs(:new).returns(mock())
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: lhm-shopify
 version: !ruby/object:Gem::Version
-  version: 4.4.1
+  version: 4.5.0
 platform: ruby
 authors:
 - SoundCloud
@@ -9,10 +9,9 @@ authors:
 - Rany Keddo
 - Tobias Bielohlawek
 - Tobias Schmidt
-autorequire:
 bindir: bin
 cert_chain: []
-date: 2024-08-21 00:00:00.000000000 Z
+date: 2025-02-07 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: retriable
@@ -182,6 +181,34 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: ostruct
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: logger
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 description: Migrate large tables without downtime by copying to a temporary table
   in chunks. The old table is not dropped. Instead, it is moved to timestamp_table_name
   for verification.
@@ -206,12 +233,10 @@ files:
 - docker-compose-mysql-5.7.yml
 - docker-compose-mysql-8.0.yml
 - docker-compose.yml
-- gemfiles/activerecord_6.1.gemfile
-- gemfiles/activerecord_6.1.gemfile.lock
-- gemfiles/activerecord_7.0.gemfile
-- gemfiles/activerecord_7.0.gemfile.lock
-- gemfiles/activerecord_7.1.gemfile
-- gemfiles/activerecord_7.1.gemfile.lock
+- gemfiles/activerecord_7.2.gemfile
+- gemfiles/activerecord_7.2.gemfile.lock
+- gemfiles/activerecord_8.0.gemfile
+- gemfiles/activerecord_8.0.gemfile.lock
 - gemfiles/activerecord_head.gemfile
 - gemfiles/activerecord_head.gemfile.lock
 - lhm.gemspec
@@ -263,6 +288,7 @@ files:
 - spec/fixtures/custom_primary_key_dest.ddl
 - spec/fixtures/destination.ddl
 - spec/fixtures/lines.ddl
+- spec/fixtures/myisam_users.ddl
 - spec/fixtures/origin.ddl
 - spec/fixtures/permissions.ddl
 - spec/fixtures/small_table.ddl
@@ -312,7 +338,6 @@ licenses:
 - BSD-3-Clause
 metadata:
   allowed_push_host: https://rubygems.org
-post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -327,8 +352,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.5.17
-signing_key:
+rubygems_version: 3.6.3
 specification_version: 4
 summary: online schema changer for mysql
 test_files: []

data/gemfiles/activerecord_7.0.gemfile

@@ -1,8 +0,0 @@
-# This file was generated by Appraisal
-
-source "https://rubygems.org"
-
-gem "activerecord", "7.0.8"
-gem "activerecord-trilogy-adapter"
-
-gemspec path: "../"

data/gemfiles/activerecord_7.0.gemfile.lock

@@ -1,74 +0,0 @@
-PATH
-  remote: ..
-  specs:
-    lhm-shopify (4.4.1)
-      retriable (>= 3.0.0)
-
-GEM
-  remote: https://rubygems.org/
-  specs:
-    activemodel (7.0.8)
-      activesupport (= 7.0.8)
-    activerecord (7.0.8)
-      activemodel (= 7.0.8)
-      activesupport (= 7.0.8)
-    activerecord-trilogy-adapter (3.1.2)
-      activerecord (>= 6.0.a, < 7.1.a)
-      trilogy (>= 2.4.0)
-    activesupport (7.0.8)
-      concurrent-ruby (~> 1.0, >= 1.0.2)
-      i18n (>= 1.6, < 2)
-      minitest (>= 5.1)
-      tzinfo (~> 2.0)
-    after_do (0.4.0)
-    appraisal (2.5.0)
-      bundler
-      rake
-      thor (>= 0.14.0)
-    byebug (11.1.3)
-    concurrent-ruby (1.2.2)
-    docile (1.4.0)
-    i18n (1.14.1)
-      concurrent-ruby (~> 1.0)
-    minitest (5.22.2)
-    mocha (2.1.0)
-      ruby2_keywords (>= 0.0.5)
-    mysql2 (0.5.5)
-    rake (13.0.6)
-    retriable (3.1.2)
-    ruby2_keywords (0.0.5)
-    simplecov (0.22.0)
-      docile (~> 1.1)
-      simplecov-html (~> 0.11)
-      simplecov_json_formatter (~> 0.1)
-    simplecov-html (0.12.3)
-    simplecov_json_formatter (0.1.4)
-    thor (1.2.2)
-    toxiproxy (2.0.2)
-    trilogy (2.6.0)
-    tzinfo (2.0.6)
-      concurrent-ruby (~> 1.0)
-
-PLATFORMS
-  arm64-darwin-21
-  arm64-darwin-22
-  x86_64-darwin-20
-  x86_64-linux
-
-DEPENDENCIES
-  activerecord (= 7.0.8)
-  activerecord-trilogy-adapter
-  after_do
-  appraisal
-  byebug
-  lhm-shopify!
-  minitest
-  mocha
-  mysql2
-  rake
-  simplecov
-  toxiproxy
-  trilogy
-
-BUNDLED WITH
-   2.2.22