lhm-shopify 4.2.1 → 4.5.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: dec22d00a0a019706b1a6d62e85e416caf210aa2d0b4f77bb3ccf562f1db8fb2
-  data.tar.gz: 181acf59e61b100dc3320b6674a7cd13a2a1dcf767c7259a88a58dbf4b826bf5
+  metadata.gz: f6810104b2e7de311501c58b38b5ab605bf808b2391e9406901dd97394ae29c2
+  data.tar.gz: 251bdd83425a0986d5d3cdded7fc05e4d53731eb86269a3059f2524780287b6e
 SHA512:
-  metadata.gz: 8c197de7b7658cdb9c31dd91adde7fdaf54505bc7e910812d8c80edf447f6266630c08a80d773391071c518f72b9527280e1ece7e2bdc00bc60d5a8c22f294ed
-  data.tar.gz: 6f1f7da4e1bd2e61fb8921eb9e724408d01ee576d98eb8ba393c2c2dada846368ec2a187e5c61d77f3691290edc815addd0e93acb8e1fa065c206564243ef798
+  metadata.gz: 866539e9bcac6acc7740766524d52e87de8572bde63d336b4dd1636c87da876d178729eaa5e716dafed53339c903ba79677980a0f29346c8ad36e0cebb145e7f
+  data.tar.gz: 0c7840100abf5b849e6750061f269fab888633bba7afd287cad3461e70e941f26f9a7a42fd215effa89797c7841c0b1200aab69d44d1119c79951ef53e8dee98

data/.github/dependabot.yml

@@ -0,0 +1,20 @@
+version: 2
+registries:
+  ruby-shopify:
+    type: rubygems-server
+    url: https://pkgs.shopify.io/basic/gems/ruby
+    username: ${{secrets.RUBYGEMS_SERVER_PKGS_SHOPIFY_IO_USERNAME}}
+    password: ${{secrets.RUBYGEMS_SERVER_PKGS_SHOPIFY_IO_PASSWORD}}
+  github-com:
+    type: git
+    url: https://github.com
+    username: ${{secrets.DEPENDENCIES_GITHUB_USER}}
+    password: ${{secrets.DEPENDENCIES_GITHUB_TOKEN}}
+updates:
+  - package-ecosystem: bundler
+    directory: "/"
+    schedule:
+      interval: daily
+    open-pull-requests-limit: 100
+    insecure-external-code-execution: allow
+    registries: "*"

data/.github/workflows/test.yml

@@ -13,16 +13,21 @@ jobs:
 
     strategy:
       fail-fast: false
+      max-parallel: 8
       matrix:
-        activerecord: ["6.1", "7.0", "7.1"]
-        ruby: ["3.0", "3.1", "3.2", "head"]
+        activerecord: ["7.2", "8.0", "head"]
+        ruby: ["3.1", "3.2", "3.3", "head"]
         mysql: ["5.7", "8.0"]
         adapter: ["mysql2", "trilogy"]
         exclude:
           - activerecord: 6.1
             ruby: head
-          - activerecord: 7.0
+          - activerecord: 7.2
             ruby: head
+          - activerecord: 8.0
+            ruby: 3.1
+          - activerecord: head
+            ruby: 3.1
 
     env:
       BUNDLE_GEMFILE: "${{ github.workspace }}/gemfiles/activerecord_${{ matrix.activerecord }}.gemfile"
@@ -40,7 +45,7 @@ jobs:
 
       - name: Setup MySQL and ProxySQL (docker-compose)
         # Might have to change to docker compose up -d (i.e. Compose V2) when the Ubuntu image changes the docker-compose version
-        run: docker-compose -f docker-compose-mysql-${{ matrix.mysql }}.yml up -d
+        run: docker compose -f docker-compose-mysql-${{ matrix.mysql }}.yml up -d
 
       - name: Wait until DBs are alive
         run: ./scripts/helpers/wait-for-dbs.sh

data/.gitignore

@@ -2,7 +2,6 @@
 .bundle
 pkg/*
 .rvmrc
-.ruby-version
 .ruby-gemset
 bin/rake
 gemfiles/vendor

data/.ruby-version

@@ -0,0 +1 @@
+3.3.6

data/Appraisals

@@ -1,13 +1,11 @@
-appraise "activerecord-6.1" do
-  gem "activerecord", "6.1.0"
-  gem "activerecord-trilogy-adapter"
+appraise "activerecord-7.2" do
+  gem "activerecord", "7.2.2.1"
 end
 
-appraise "activerecord-7.0" do
-  gem "activerecord", "7.0.8"
-  gem "activerecord-trilogy-adapter"
+appraise "activerecord-8.0" do
+  gem "activerecord", "8.0.1"
 end
 
-appraise "activerecord-7.1" do
-  gem "activerecord", "7.1.1"
+appraise "activerecord-head" do
+  gem "activerecord", git: "https://github.com/rails/rails.git", branch: "main"
 end

data/CHANGELOG.md

@@ -1,5 +1,34 @@
 # Unreleased
 
+# 4.5.1 (Jul, 2025)
+* Create update before insert trigger
+
+# 4.5.0 (Feb, 2025)
+* Update test matrix to include Rails 8
+* Add option to force tables to use default engine
+* Change default algorithm for add/rename/remove column operations
+* Drop support for ActiveRecord version 6.1, as it has reached EOL
+* Add support for tables with generated columns
+
+# 4.4.2 (Sep, 2024)
+* Allow caller to set the algorithm that will be used for DDL ALTER TABLE operations
+
+# 4.4.1 (Aug, 2024)
+* Extend max_binlog_cache_size exceeded error handling to all throttlers
+
+# 4.4.0 (Aug, 2024)
+* Add support for retrying chunks when running into max_binlog_cache_size exceeded error
+
+# 4.3.0 (Aug, 2024)
+* Drop support for Ruby 3.0, as it reached its EOL
+* Add support for next Rails version
+
+# 4.2.3 (Jul, 2024)
+* Fix check for warnings against PKs with line breaks
+
+# 4.2.2 (Jun, 2024)
+* Avoid using the INSTANT algorithm.
+
 # 4.2.1 (Mar, 2024)
 * Retry more errors when using Trilogy.
 

data/Gemfile.lock

@@ -1,67 +1,74 @@
 PATH
   remote: .
   specs:
-    lhm-shopify (4.2.1)
+    lhm-shopify (4.5.1)
       retriable (>= 3.0.0)
 
 GEM
   remote: https://rubygems.org/
   specs:
-    activemodel (7.1.1)
-      activesupport (= 7.1.1)
-    activerecord (7.1.1)
-      activemodel (= 7.1.1)
-      activesupport (= 7.1.1)
+    activemodel (8.0.1)
+      activesupport (= 8.0.1)
+    activerecord (8.0.1)
+      activemodel (= 8.0.1)
+      activesupport (= 8.0.1)
       timeout (>= 0.4.0)
-    activesupport (7.1.1)
+    activesupport (8.0.1)
       base64
+      benchmark (>= 0.3)
       bigdecimal
-      concurrent-ruby (~> 1.0, >= 1.0.2)
+      concurrent-ruby (~> 1.0, >= 1.3.1)
       connection_pool (>= 2.2.5)
       drb
       i18n (>= 1.6, < 2)
+      logger (>= 1.4.2)
       minitest (>= 5.1)
-      mutex_m
-      tzinfo (~> 2.0)
+      securerandom (>= 0.3)
+      tzinfo (~> 2.0, >= 2.0.5)
+      uri (>= 0.13.1)
     after_do (0.4.0)
     appraisal (2.5.0)
       bundler
       rake
       thor (>= 0.14.0)
-    base64 (0.1.1)
-    bigdecimal (3.1.4)
+    base64 (0.2.0)
+    benchmark (0.4.0)
+    bigdecimal (3.1.9)
     byebug (11.1.3)
-    concurrent-ruby (1.2.2)
-    connection_pool (2.4.1)
-    docile (1.4.0)
-    drb (2.1.1)
-      ruby2_keywords
-    i18n (1.14.1)
+    concurrent-ruby (1.3.5)
+    connection_pool (2.5.0)
+    docile (1.4.1)
+    drb (2.2.1)
+    i18n (1.14.7)
       concurrent-ruby (~> 1.0)
-    minitest (5.22.2)
-    mocha (2.1.0)
+    logger (1.6.5)
+    minitest (5.25.4)
+    mocha (2.4.5)
       ruby2_keywords (>= 0.0.5)
-    mutex_m (0.1.2)
-    mysql2 (0.5.5)
-    rake (13.0.6)
+    mysql2 (0.5.6)
+    ostruct (0.6.1)
+    rake (13.2.1)
     retriable (3.1.2)
     ruby2_keywords (0.0.5)
+    securerandom (0.4.1)
     simplecov (0.22.0)
       docile (~> 1.1)
       simplecov-html (~> 0.11)
       simplecov_json_formatter (~> 0.1)
     simplecov-html (0.12.3)
     simplecov_json_formatter (0.1.4)
-    thor (1.2.2)
-    timeout (0.4.0)
+    thor (1.3.2)
+    timeout (0.4.3)
     toxiproxy (2.0.2)
-    trilogy (2.6.0)
+    trilogy (2.8.1)
     tzinfo (2.0.6)
       concurrent-ruby (~> 1.0)
+    uri (1.0.2)
 
 PLATFORMS
   arm64-darwin-21
   arm64-darwin-22
+  arm64-darwin-23
   x86_64-darwin-20
   x86_64-linux
 
@@ -71,13 +78,15 @@ DEPENDENCIES
   appraisal
   byebug
   lhm-shopify!
+  logger
   minitest
   mocha
   mysql2
+  ostruct
   rake
   simplecov
   toxiproxy
   trilogy
 
 BUNDLED WITH
-   2.2.22
+   2.6.3

data/README.md

@@ -207,6 +207,35 @@ Or to set that as default throttler, use the following (for instance in a Rails
 Lhm.setup_throttler(:threads_running_throttler)
 ```
 
+### Retrying chunks using Throttler 
+
+If chunks fail due to the MySQL `max_binlog_cache_size` being exceeded, the Chunker class will call the `backoff_stride` method on the throttler to reduce the stride size and retry the chunk. The default backoff factor is 0.2 (reducing the chunk size by 20% with each failure), and the minimum stride size is 1 (the stride can be reduced until each chunk is 1 row). Default values can be overridden (see below). To disable backoff, set `min_stride_size` equal to `stride_size`; this will throw an exception when `max_binlog_cache_size` is exceeded. Configuration options should be passed into the time throttler constructor or in the `throttler_options` of change_table when using a time throttler.
+
+```ruby
+## Configure on throttler instance 
+my_throttler = Lhm::Throttler::ThreadsRunning.new(stride: 2000, delay: 1, backoff_reduction_factor: 0.1, min_stride_size: 10)
+
+Lhm.change_table :users, throttler: my_throttler  do |m|
+  ...
+end
+
+## Or pass as throttler options 
+Lhm.change_table :users, {
+  throttler: :time_throttler,
+  throttler_options: {
+    stride: 2000,
+    delay: 1,
+    backoff_reduction_factor: 0.1,
+    min_stride_size: 10
+  }
+} do |t|
+  ...
+end
+```
+
+### Custom Throttlers with retry functionality 
+If using your own throttler, ensure that the class definition includes the `Lhm::Throttler::BackoffReduction` to retry failed chunks.  
+
 ## Table rename strategies
 
 There are two different table rename strategies available: `LockedSwitcher` and
@@ -311,7 +340,7 @@ open coverage/index.html
 ```
 
 ### Merging for a new version
-When creating a PR for a new version, make sure that th version has been bumped in `lib/lhm/version.rb`. Then run the following code snippet to ensure the everything is consistent, otherwise
+When creating a PR for a new version, make sure that the version has been bumped in `lib/lhm/version.rb`. Then run the following code snippet to ensure the everything is consistent, otherwise
 the gem will not publish.
 ```bash
 bundle install

data/dev.yml

@@ -1,11 +1,9 @@
 name: lhm
 up:
-  - homebrew:
-      - mysql-client@5.7:
-          or: [mysql@5.7]
-          conflicts: [shopify/shopify/mysql-client, mysql-connector-c, mysql, mysql-client]
+  - packages:
+      - mysql-client
       - wget
-  - ruby: 3.2.2
+  - ruby
   - bundler
   - podman
   - custom:
@@ -14,8 +12,9 @@ up:
       meet: ":"
   - custom:
       name: Podman compose
-      met?: podman-compose ps | grep -ioE -q "lhm.*running\(4\)"
-      meet: podman-compose up -d
+      met?: "[[ $(podman-compose ps | grep -c -e 'running' -e 'Up') == 4 ]] 2> /dev/null"
+      meet: podman-compose -f docker-compose-mysql-8.0.yml up -d
+      down: podman-compose down
   - custom:
       name: Waiting for DBs to be operational
       met?: ./scripts/helpers/wait-for-dbs.sh

data/gemfiles/{activerecord_6.1.gemfile → activerecord_7.2.gemfile}

@@ -2,7 +2,6 @@
 
 source "https://rubygems.org"
 
-gem "activerecord", "6.1.0"
-gem "activerecord-trilogy-adapter"
+gem "activerecord", "7.2.2.1"
 
 gemspec path: "../"

data/gemfiles/{activerecord_6.1.gemfile.lock → activerecord_7.2.gemfile.lock}

@@ -1,76 +1,90 @@
 PATH
   remote: ..
   specs:
-    lhm-shopify (4.2.1)
+    lhm-shopify (4.5.1)
       retriable (>= 3.0.0)
 
 GEM
   remote: https://rubygems.org/
   specs:
-    activemodel (6.1.0)
-      activesupport (= 6.1.0)
-    activerecord (6.1.0)
-      activemodel (= 6.1.0)
-      activesupport (= 6.1.0)
-    activerecord-trilogy-adapter (3.1.2)
-      activerecord (>= 6.0.a, < 7.1.a)
-      trilogy (>= 2.4.0)
-    activesupport (6.1.0)
-      concurrent-ruby (~> 1.0, >= 1.0.2)
+    activemodel (7.2.2.1)
+      activesupport (= 7.2.2.1)
+    activerecord (7.2.2.1)
+      activemodel (= 7.2.2.1)
+      activesupport (= 7.2.2.1)
+      timeout (>= 0.4.0)
+    activesupport (7.2.2.1)
+      base64
+      benchmark (>= 0.3)
+      bigdecimal
+      concurrent-ruby (~> 1.0, >= 1.3.1)
+      connection_pool (>= 2.2.5)
+      drb
       i18n (>= 1.6, < 2)
+      logger (>= 1.4.2)
       minitest (>= 5.1)
-      tzinfo (~> 2.0)
-      zeitwerk (~> 2.3)
+      securerandom (>= 0.3)
+      tzinfo (~> 2.0, >= 2.0.5)
     after_do (0.4.0)
     appraisal (2.5.0)
       bundler
       rake
       thor (>= 0.14.0)
+    base64 (0.2.0)
+    benchmark (0.4.0)
+    bigdecimal (3.1.9)
     byebug (11.1.3)
-    concurrent-ruby (1.2.2)
+    concurrent-ruby (1.3.5)
+    connection_pool (2.5.0)
     docile (1.4.0)
-    i18n (1.14.1)
+    drb (2.2.1)
+    i18n (1.14.7)
       concurrent-ruby (~> 1.0)
-    minitest (5.22.2)
+    logger (1.6.5)
+    minitest (5.25.4)
     mocha (2.1.0)
       ruby2_keywords (>= 0.0.5)
     mysql2 (0.5.5)
+    ostruct (0.6.1)
     rake (13.0.6)
     retriable (3.1.2)
     ruby2_keywords (0.0.5)
+    securerandom (0.4.1)
     simplecov (0.22.0)
       docile (~> 1.1)
       simplecov-html (~> 0.11)
       simplecov_json_formatter (~> 0.1)
     simplecov-html (0.12.3)
     simplecov_json_formatter (0.1.4)
-    thor (1.2.2)
+    thor (1.3.2)
+    timeout (0.4.3)
     toxiproxy (2.0.2)
-    trilogy (2.6.0)
+    trilogy (2.9.0)
     tzinfo (2.0.6)
       concurrent-ruby (~> 1.0)
-    zeitwerk (2.6.12)
 
 PLATFORMS
   arm64-darwin-21
   arm64-darwin-22
+  arm64-darwin-23
   x86_64-darwin-20
   x86_64-linux
 
 DEPENDENCIES
-  activerecord (= 6.1.0)
-  activerecord-trilogy-adapter
+  activerecord (= 7.2.2.1)
   after_do
   appraisal
   byebug
   lhm-shopify!
+  logger
   minitest
   mocha
   mysql2
+  ostruct
   rake
   simplecov
   toxiproxy
   trilogy
 
 BUNDLED WITH
-   2.2.22
+   2.6.3

data/gemfiles/{activerecord_7.1.gemfile → activerecord_8.0.gemfile}

@@ -2,6 +2,6 @@
 
 source "https://rubygems.org"
 
-gem "activerecord", "7.1.1"
+gem "activerecord", "8.0.1"
 
 gemspec path: "../"

data/gemfiles/{activerecord_7.1.gemfile.lock → activerecord_8.0.gemfile.lock}

@@ -1,83 +1,92 @@
 PATH
   remote: ..
   specs:
-    lhm-shopify (4.2.1)
+    lhm-shopify (4.5.1)
       retriable (>= 3.0.0)
 
 GEM
   remote: https://rubygems.org/
   specs:
-    activemodel (7.1.1)
-      activesupport (= 7.1.1)
-    activerecord (7.1.1)
-      activemodel (= 7.1.1)
-      activesupport (= 7.1.1)
+    activemodel (8.0.1)
+      activesupport (= 8.0.1)
+    activerecord (8.0.1)
+      activemodel (= 8.0.1)
+      activesupport (= 8.0.1)
       timeout (>= 0.4.0)
-    activesupport (7.1.1)
+    activesupport (8.0.1)
       base64
+      benchmark (>= 0.3)
       bigdecimal
-      concurrent-ruby (~> 1.0, >= 1.0.2)
+      concurrent-ruby (~> 1.0, >= 1.3.1)
       connection_pool (>= 2.2.5)
       drb
       i18n (>= 1.6, < 2)
+      logger (>= 1.4.2)
       minitest (>= 5.1)
-      mutex_m
-      tzinfo (~> 2.0)
+      securerandom (>= 0.3)
+      tzinfo (~> 2.0, >= 2.0.5)
+      uri (>= 0.13.1)
     after_do (0.4.0)
     appraisal (2.5.0)
       bundler
       rake
       thor (>= 0.14.0)
-    base64 (0.1.1)
-    bigdecimal (3.1.4)
+    base64 (0.2.0)
+    benchmark (0.4.0)
+    bigdecimal (3.1.9)
     byebug (11.1.3)
-    concurrent-ruby (1.2.2)
-    connection_pool (2.4.1)
+    concurrent-ruby (1.3.5)
+    connection_pool (2.5.0)
     docile (1.4.0)
-    drb (2.1.1)
-      ruby2_keywords
+    drb (2.2.1)
     i18n (1.14.1)
       concurrent-ruby (~> 1.0)
+    logger (1.6.1)
     minitest (5.22.2)
     mocha (2.1.0)
       ruby2_keywords (>= 0.0.5)
-    mutex_m (0.1.2)
     mysql2 (0.5.5)
+    ostruct (0.6.1)
     rake (13.0.6)
     retriable (3.1.2)
     ruby2_keywords (0.0.5)
+    securerandom (0.4.1)
     simplecov (0.22.0)
       docile (~> 1.1)
       simplecov-html (~> 0.11)
       simplecov_json_formatter (~> 0.1)
     simplecov-html (0.12.3)
     simplecov_json_formatter (0.1.4)
-    thor (1.2.2)
-    timeout (0.4.0)
+    thor (1.3.2)
+    timeout (0.4.3)
     toxiproxy (2.0.2)
-    trilogy (2.6.0)
+    trilogy (2.9.0)
     tzinfo (2.0.6)
       concurrent-ruby (~> 1.0)
+    uri (1.0.2)
 
 PLATFORMS
   arm64-darwin-21
   arm64-darwin-22
+  arm64-darwin-23
   x86_64-darwin-20
   x86_64-linux
 
 DEPENDENCIES
-  activerecord (= 7.1.1)
+  activerecord (= 8.0.1)
   after_do
   appraisal
   byebug
   lhm-shopify!
+  logger
   minitest
   mocha
   mysql2
+  ostruct
   rake
   simplecov
   toxiproxy
   trilogy
 
 BUNDLED WITH
-   2.2.22
+   2.6.3

data/gemfiles/{activerecord_7.0.gemfile → activerecord_head.gemfile}

@@ -2,7 +2,6 @@
 
 source "https://rubygems.org"
 
-gem "activerecord", "7.0.8"
-gem "activerecord-trilogy-adapter"
+gem "activerecord", git: "https://github.com/rails/rails.git", branch: "main"
 
 gemspec path: "../"

data/gemfiles/activerecord_head.gemfile.lock

@@ -0,0 +1,96 @@
+GIT
+  remote: https://github.com/rails/rails.git
+  revision: bdad2707835949a96dd29e747b8c324593efd4d8
+  branch: main
+  specs:
+    activemodel (8.1.0.alpha)
+      activesupport (= 8.1.0.alpha)
+    activerecord (8.1.0.alpha)
+      activemodel (= 8.1.0.alpha)
+      activesupport (= 8.1.0.alpha)
+      timeout (>= 0.4.0)
+    activesupport (8.1.0.alpha)
+      base64
+      benchmark (>= 0.3)
+      bigdecimal
+      concurrent-ruby (~> 1.0, >= 1.3.1)
+      connection_pool (>= 2.2.5)
+      drb
+      i18n (>= 1.6, < 2)
+      logger (>= 1.4.2)
+      minitest (>= 5.1)
+      securerandom (>= 0.3)
+      tzinfo (~> 2.0, >= 2.0.5)
+      uri (>= 0.13.1)
+
+PATH
+  remote: ..
+  specs:
+    lhm-shopify (4.5.1)
+      retriable (>= 3.0.0)
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    after_do (0.4.0)
+    appraisal (2.5.0)
+      bundler
+      rake
+      thor (>= 0.14.0)
+    base64 (0.2.0)
+    benchmark (0.4.0)
+    bigdecimal (3.1.9)
+    byebug (11.1.3)
+    concurrent-ruby (1.3.5)
+    connection_pool (2.5.0)
+    docile (1.4.1)
+    drb (2.2.1)
+    i18n (1.14.7)
+      concurrent-ruby (~> 1.0)
+    logger (1.6.5)
+    minitest (5.25.4)
+    mocha (2.4.5)
+      ruby2_keywords (>= 0.0.5)
+    mysql2 (0.5.6)
+    ostruct (0.6.1)
+    rake (13.2.1)
+    retriable (3.1.2)
+    ruby2_keywords (0.0.5)
+    securerandom (0.4.1)
+    simplecov (0.22.0)
+      docile (~> 1.1)
+      simplecov-html (~> 0.11)
+      simplecov_json_formatter (~> 0.1)
+    simplecov-html (0.12.3)
+    simplecov_json_formatter (0.1.4)
+    thor (1.3.2)
+    timeout (0.4.3)
+    toxiproxy (2.0.2)
+    trilogy (2.9.0)
+    tzinfo (2.0.6)
+      concurrent-ruby (~> 1.0)
+    uri (1.0.2)
+
+PLATFORMS
+  arm64-darwin-22
+  arm64-darwin-23
+  x86_64-linux
+
+DEPENDENCIES
+  activerecord!
+  after_do
+  appraisal
+  byebug
+  lhm-shopify!
+  logger
+  minitest
+  mocha
+  mysql2
+  ostruct
+  rake
+  simplecov
+  toxiproxy
+  trilogy
+
+BUNDLED WITH
+   2.6.3