lhm-shopify 3.5.4 → 4.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 4f676427b10052113b08c4dda8f648bc11234024deab2f1e96de332e0906a110
-  data.tar.gz: dbd91d1b72d713e9d6e18edb46900971ee89c20563d5e93db9eec746887a12d7
+  metadata.gz: 34c64cb93bda1fe53758ea181451438890d8460e167784dfff3255aba7ccf662
+  data.tar.gz: 14e18340ed78985984b93c8d36328ccb9a29d1318639e190ad98f5ceecdf828c
 SHA512:
-  metadata.gz: be5dc9f5d6d2a3ae62b476fa19455bd0f864b989d6ffbff35a93a0afea4e8d9be4492a4b3bfc2bec1029193114187eaf4eda5bb6dfde9d0be34884bde317c1c1
-  data.tar.gz: aa4e866f74da3255586c1597ae63a3792606bd092e46c3a90335d22b9f6c9b243bc133d99e50233c884b03064dfaeee75983597f84733484e05ec456833f3395
+  metadata.gz: 4f83e952632a11d868a75da5d9b2e64b7326090ed0b37f74d221f8b86a1345bef783c5d56ca82691d13067b410cf0926d2e88be0939c9cb925b66f418e4fcb9e
+  data.tar.gz: 6aefea0808ed1ba537d550464492784f9b2cc504858a2524cf7c74bde16123c4be0eb0965c206f8f3cb0c2cceca48bc17651d111e8d9401a7f3c762efbcef7a3

data/.github/workflows/test.yml

@@ -15,14 +15,14 @@ jobs:
       fail-fast: false
       matrix:
         include:
-          - ruby-version: "2.4"
-            activerecord-version: "5.2"
-          - ruby-version: "2.5"
+          - ruby-version: "3.0"
             activerecord-version: "6.0"
-          - ruby-version: "2.7.5"
+          - ruby-version: "3.1"
             activerecord-version: "6.1"
-          - ruby-version: "3.0"
-            activerecord-version: "7.0.0.alpha2"
+          - ruby-version: "3.2"
+            activerecord-version: "7.0"
+          - ruby-version: "3.2"
+            activerecord-version: "7.1.0.beta1"
     steps:
       - uses: actions/checkout@v2
       - name: Set up Ruby

data/Appraisals

@@ -1,16 +1,6 @@
-# Oldest supported Rails version
-appraise "activerecord-5.2" do
-  gem "activerecord", "5.2.6"
-
-# This gemfile will be ran against ruby 2.4 and some gems might require 2.4+. The following list are the gems
-# and their respective versions that work with ruby <= 2.4
-  gem "simplecov", "0.18.5"
-  gem "docile", "1.3.5"
-end
-
 # First conflicted version
 appraise "activerecord-6.0" do
-  gem "activerecord", "6.0.0"
+  gem "activerecord", "6.0.3"
 end
 
 # Second conflicted version
@@ -19,6 +9,11 @@ appraise "activerecord-6.1" do
 end
 
 # Latest version at the moment
-appraise "activerecord-7.0.0.alpha2" do
-  gem "activerecord", "7.0.0.alpha2"
+appraise "activerecord-7.0" do
+  gem "activerecord", "7.0.8"
+end
+
+# Next release
+appraise "activerecord-7.1.0.beta1" do
+  gem "activerecord", "7.1.0.beta1"
 end

data/CHANGELOG.md

@@ -1,3 +1,16 @@
+# Unreleased
+
+# 4.0.0 (Sep, 2023)
+* Deprecate `SlaveLag` throttler class name. Use `ReplicaLag` instead (https://github.com/Shopify/lhm/pull/144)
+* Deprecate `slave_lag_throttler` throttler config value. Use `replica_lag_throttler` instead (https://github.com/Shopify/lhm/pull/144)
+* Fix errors when creating indexes with whitespace between column names and sizes. (https://github.com/Shopify/lhm/pull/145)
+* Test against Ruby 3.2 and Rails 7.1.0.beta1. (https://github.com/Shopify/lhm/pull/146)
+* Drop support for Ruby 2 and Rails 5. (https://github.com/Shopify/lhm/pull/148)
+* Fix thread throttler #stride API. (https://github.com/Shopify/lhm/pull/131)
+
+# 3.5.5 (Jan, 2022)
+* Fix error where from Config shadowing which would cause LHM to abort on reconnect (https://github.com/Shopify/lhm/pull/128)
+
 # 3.5.4 (Dec, 2021)
 * Refactored the way options are handled internally. Code is now much clearer to understand
 * Removed optional connection_options from `Lhm.setup` and `Lhm.connection`

data/Gemfile.lock

@@ -1,51 +1,53 @@
 PATH
   remote: .
   specs:
-    lhm-shopify (3.5.4)
+    lhm-shopify (4.0.0)
       retriable (>= 3.0.0)
 
 GEM
   remote: https://rubygems.org/
   specs:
-    activemodel (6.1.4.1)
-      activesupport (= 6.1.4.1)
-    activerecord (6.1.4.1)
-      activemodel (= 6.1.4.1)
-      activesupport (= 6.1.4.1)
-    activesupport (6.1.4.1)
+    activemodel (7.0.8)
+      activesupport (= 7.0.8)
+    activerecord (7.0.8)
+      activemodel (= 7.0.8)
+      activesupport (= 7.0.8)
+    activesupport (7.0.8)
       concurrent-ruby (~> 1.0, >= 1.0.2)
       i18n (>= 1.6, < 2)
       minitest (>= 5.1)
       tzinfo (~> 2.0)
-      zeitwerk (~> 2.3)
     after_do (0.4.0)
-    appraisal (2.4.1)
+    appraisal (2.5.0)
       bundler
       rake
       thor (>= 0.14.0)
     byebug (11.1.3)
-    concurrent-ruby (1.1.9)
+    concurrent-ruby (1.2.2)
     docile (1.4.0)
-    i18n (1.8.11)
+    i18n (1.14.1)
       concurrent-ruby (~> 1.0)
-    minitest (5.14.4)
-    mocha (1.13.0)
-    mysql2 (0.5.3)
+    minitest (5.20.0)
+    mocha (2.1.0)
+      ruby2_keywords (>= 0.0.5)
+    mysql2 (0.5.5)
     rake (13.0.6)
     retriable (3.1.2)
-    simplecov (0.21.2)
+    ruby2_keywords (0.0.5)
+    simplecov (0.22.0)
       docile (~> 1.1)
       simplecov-html (~> 0.11)
       simplecov_json_formatter (~> 0.1)
     simplecov-html (0.12.3)
-    simplecov_json_formatter (0.1.3)
-    thor (1.1.0)
-    toxiproxy (2.0.0)
-    tzinfo (2.0.4)
+    simplecov_json_formatter (0.1.4)
+    thor (1.2.2)
+    toxiproxy (2.0.2)
+    tzinfo (2.0.6)
       concurrent-ruby (~> 1.0)
-    zeitwerk (2.5.1)
 
 PLATFORMS
+  arm64-darwin-21
+  arm64-darwin-22
   x86_64-darwin-20
   x86_64-linux
 

data/README.md

@@ -57,7 +57,7 @@ a single integer numeric key column named `id`.
 
 ## Installation
 
-Install it via `gem install lhm` or by adding `gem "lhm"` to your `Gemfile`.
+Install it via `gem install lhm-shopify` or by adding `gem "lhm-shopify"` to your `Gemfile`.
 
 ## Usage
 
@@ -165,13 +165,13 @@ Lhm.change_table :users, throttler: my_throttler  do |m|
 end
 ```
 
-### SlaveLag Throttler
+### ReplicaLag Throttler
 
 Lhm uses by default the time throttler, however a better solution is to throttle the copy of the data
-depending on the time that the slaves are behind. To use the SlaveLag throttler:
+depending on the time that the replicas are behind. To use the ReplicaLag throttler:
 
 ```ruby
-Lhm.change_table :users, throttler: :slave_lag_throttler  do |m|
+Lhm.change_table :users, throttler: :replica_lag_throttler  do |m|
   ...
 end
 ```
@@ -179,7 +179,7 @@ end
 Or to set that as default throttler, use the following (for instance in a Rails initializer):
 
 ```ruby
-Lhm.setup_throttler(:slave_lag_throttler)
+Lhm.setup_throttler(:replica_lag_throttler)
 ```
 
 ### ThreadsRunning Throttler
@@ -189,6 +189,13 @@ throttle based on the number of threads running in MySQL, as a proxy for "is
 this operation causing excessive load":
 
 ```ruby
+my_throttler = Lhm::Throttler::ThreadsRunning.new(stride: 100_000)
+
+Lhm.change_table :users, throttler: my_throttler  do |m|
+  ...
+end
+
+# or use default settings:
 Lhm.change_table :users, throttler: :threads_running_throttler do |m|
   ...
 end
@@ -312,7 +319,7 @@ bundle update
 bundle exec appraisals install
 ```
 
-### Docker Compose
+### Podman Compose
 The integration tests rely on a replication configuration for MySQL which is being proxied by an instance of ProxySQL.
 It is important that every container is running to execute the integration test suite.
 
@@ -327,7 +334,7 @@ The license is included as [LICENSE](LICENSE) in this directory.
   * [Twitter][2]
   * [pt-online-schema-change][3]
 
-[0]: http://openarkkit.googlecode.com
+[0]: https://shlomi-noach.github.io/openarkkit/introduction.html
 [1]: http://www.facebook.com/note.php?note\_id=430801045932
 [2]: https://github.com/freels/table_migrator
 [3]: http://www.percona.com/doc/percona-toolkit/2.1/pt-online-schema-change.html

data/dev.yml

@@ -5,17 +5,17 @@ up:
           or: [mysql@5.7]
           conflicts: [shopify/shopify/mysql-client, mysql-connector-c, mysql, mysql-client]
       - wget
-  - ruby: 2.7.5
+  - ruby: 3.2.2
   - bundler
+  - podman
   - custom:
       name: Get Appraisal gems
       met?: bundle exec appraisal install
       meet: ":"
-  - docker
   - custom:
-      name: Docker Compose
-      met?: docker compose ls | grep -ioE -q "lhm.*running\(4\)"
-      meet: docker compose up -d
+      name: Podman compose
+      met?: podman-compose ls | grep -ioE -q "lhm.*running\(4\)"
+      meet: podman-compose up -d
   - custom:
       name: Waiting for DBs to be operational
       met?: ./scripts/helpers/wait-for-dbs.sh
@@ -25,7 +25,8 @@ commands:
   unit: bundle exec rake unit
   int: bundle exec rake integration
   test:
-    optional: file
+    syntax:
+      optional: file
     aliases: [ t ]
     run: |
       if [[ $# -eq 0 ]]; then
@@ -37,6 +38,9 @@ commands:
   cov: rm -rf coverage; COV=1 bundle exec rake unit && bundle exec rake integration; open coverage/index.html
   logs:
     desc: "See the DB logs (ctrl-c + ctrl-c to exit)"
-    run: docker-compose logs -f
+    run: podman-compose logs -f
   clear:
-    run: docker-compose rm -v -s -f && docker-compose up -d && ./scripts/helpers/wait-for-dbs.sh
+    run: podman-compose down -v && podman-compose up -d && ./scripts/helpers/wait-for-dbs.sh
+  pre-publish:
+    # Ensures all Gemfile.lock are sync with the new version in `lhm/version.rb` and runs appraisals
+    run: bundle install && bundle exec appraisal install && bundle exec appraisal rake specs

data/docker-compose.yml

@@ -3,6 +3,7 @@ services:
   mysql-1:
     container_name: mysql-1
     image: percona:5.7
+    platform: linux/amd64
     command:
       --server-id=1
       --log-bin
@@ -23,6 +24,7 @@ services:
   mysql-2:
     container_name: mysql-2
     image: percona:5.7
+    platform: linux/amd64
     command:
       --server-id=2
       --log-bin

data/gemfiles/activerecord_6.0.gemfile

@@ -2,6 +2,6 @@
 
 source "https://rubygems.org"
 
-gem "activerecord", "6.0.0"
+gem "activerecord", "6.0.3"
 
 gemspec path: "../"

data/gemfiles/activerecord_6.0.gemfile.lock

@@ -1,57 +1,61 @@
 PATH
   remote: ..
   specs:
-    lhm-shopify (3.5.4)
+    lhm-shopify (4.0.0)
       retriable (>= 3.0.0)
 
 GEM
   remote: https://rubygems.org/
   specs:
-    activemodel (6.0.0)
-      activesupport (= 6.0.0)
-    activerecord (6.0.0)
-      activemodel (= 6.0.0)
-      activesupport (= 6.0.0)
-    activesupport (6.0.0)
+    activemodel (6.0.3)
+      activesupport (= 6.0.3)
+    activerecord (6.0.3)
+      activemodel (= 6.0.3)
+      activesupport (= 6.0.3)
+    activesupport (6.0.3)
       concurrent-ruby (~> 1.0, >= 1.0.2)
       i18n (>= 0.7, < 2)
       minitest (~> 5.1)
       tzinfo (~> 1.1)
-      zeitwerk (~> 2.1, >= 2.1.8)
+      zeitwerk (~> 2.2, >= 2.2.2)
     after_do (0.4.0)
-    appraisal (2.4.1)
+    appraisal (2.5.0)
       bundler
       rake
       thor (>= 0.14.0)
     byebug (11.1.3)
-    concurrent-ruby (1.1.9)
+    concurrent-ruby (1.2.2)
     docile (1.4.0)
-    i18n (1.8.11)
+    i18n (1.14.1)
       concurrent-ruby (~> 1.0)
-    minitest (5.14.4)
-    mocha (1.13.0)
-    mysql2 (0.5.3)
+    minitest (5.20.0)
+    mocha (2.1.0)
+      ruby2_keywords (>= 0.0.5)
+    mysql2 (0.5.5)
     rake (13.0.6)
     retriable (3.1.2)
-    simplecov (0.21.2)
+    ruby2_keywords (0.0.5)
+    simplecov (0.22.0)
       docile (~> 1.1)
       simplecov-html (~> 0.11)
       simplecov_json_formatter (~> 0.1)
     simplecov-html (0.12.3)
-    simplecov_json_formatter (0.1.3)
-    thor (1.1.0)
+    simplecov_json_formatter (0.1.4)
+    thor (1.2.2)
     thread_safe (0.3.6)
-    toxiproxy (2.0.0)
-    tzinfo (1.2.9)
+    toxiproxy (2.0.2)
+    tzinfo (1.2.11)
       thread_safe (~> 0.1)
-    zeitwerk (2.5.1)
+    zeitwerk (2.6.11)
 
 PLATFORMS
+  arm64-darwin-21
+  arm64-darwin-22
   x86_64-darwin-20
   x86_64-linux
 
 DEPENDENCIES
-  activerecord (= 6.0.0)
+  activerecord (= 6.0.3)
   after_do
   appraisal
   byebug

data/gemfiles/activerecord_6.1.gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: ..
   specs:
-    lhm-shopify (3.5.4)
+    lhm-shopify (4.0.0)
       retriable (>= 3.0.0)
 
 GEM
@@ -19,33 +19,37 @@ GEM
       tzinfo (~> 2.0)
       zeitwerk (~> 2.3)
     after_do (0.4.0)
-    appraisal (2.4.1)
+    appraisal (2.5.0)
       bundler
       rake
       thor (>= 0.14.0)
     byebug (11.1.3)
-    concurrent-ruby (1.1.9)
+    concurrent-ruby (1.2.2)
     docile (1.4.0)
-    i18n (1.8.11)
+    i18n (1.14.1)
       concurrent-ruby (~> 1.0)
-    minitest (5.14.4)
-    mocha (1.13.0)
-    mysql2 (0.5.3)
+    minitest (5.20.0)
+    mocha (2.1.0)
+      ruby2_keywords (>= 0.0.5)
+    mysql2 (0.5.5)
     rake (13.0.6)
     retriable (3.1.2)
-    simplecov (0.21.2)
+    ruby2_keywords (0.0.5)
+    simplecov (0.22.0)
       docile (~> 1.1)
       simplecov-html (~> 0.11)
       simplecov_json_formatter (~> 0.1)
     simplecov-html (0.12.3)
-    simplecov_json_formatter (0.1.3)
-    thor (1.1.0)
-    toxiproxy (2.0.0)
-    tzinfo (2.0.4)
+    simplecov_json_formatter (0.1.4)
+    thor (1.2.2)
+    toxiproxy (2.0.2)
+    tzinfo (2.0.6)
       concurrent-ruby (~> 1.0)
-    zeitwerk (2.5.1)
+    zeitwerk (2.6.11)
 
 PLATFORMS
+  arm64-darwin-21
+  arm64-darwin-22
   x86_64-darwin-20
   x86_64-linux
 

data/gemfiles/{activerecord_7.0.0.alpha2.gemfile → activerecord_7.0.gemfile}

@@ -2,6 +2,6 @@
 
 source "https://rubygems.org"
 
-gem "activerecord", "7.0.0.alpha2"
+gem "activerecord", "7.0.8"
 
 gemspec path: "../"

data/gemfiles/{activerecord_7.0.0.alpha2.gemfile.lock → activerecord_7.0.gemfile.lock}

@@ -1,54 +1,58 @@
 PATH
   remote: ..
   specs:
-    lhm-shopify (3.5.4)
+    lhm-shopify (4.0.0)
       retriable (>= 3.0.0)
 
 GEM
   remote: https://rubygems.org/
   specs:
-    activemodel (7.0.0.alpha2)
-      activesupport (= 7.0.0.alpha2)
-    activerecord (7.0.0.alpha2)
-      activemodel (= 7.0.0.alpha2)
-      activesupport (= 7.0.0.alpha2)
-    activesupport (7.0.0.alpha2)
+    activemodel (7.0.8)
+      activesupport (= 7.0.8)
+    activerecord (7.0.8)
+      activemodel (= 7.0.8)
+      activesupport (= 7.0.8)
+    activesupport (7.0.8)
       concurrent-ruby (~> 1.0, >= 1.0.2)
       i18n (>= 1.6, < 2)
       minitest (>= 5.1)
       tzinfo (~> 2.0)
     after_do (0.4.0)
-    appraisal (2.4.1)
+    appraisal (2.5.0)
       bundler
       rake
       thor (>= 0.14.0)
     byebug (11.1.3)
-    concurrent-ruby (1.1.9)
+    concurrent-ruby (1.2.2)
     docile (1.4.0)
-    i18n (1.8.11)
+    i18n (1.14.1)
       concurrent-ruby (~> 1.0)
-    minitest (5.14.4)
-    mocha (1.13.0)
-    mysql2 (0.5.3)
+    minitest (5.20.0)
+    mocha (2.1.0)
+      ruby2_keywords (>= 0.0.5)
+    mysql2 (0.5.5)
     rake (13.0.6)
     retriable (3.1.2)
-    simplecov (0.21.2)
+    ruby2_keywords (0.0.5)
+    simplecov (0.22.0)
       docile (~> 1.1)
       simplecov-html (~> 0.11)
       simplecov_json_formatter (~> 0.1)
     simplecov-html (0.12.3)
-    simplecov_json_formatter (0.1.3)
-    thor (1.1.0)
-    toxiproxy (2.0.0)
-    tzinfo (2.0.4)
+    simplecov_json_formatter (0.1.4)
+    thor (1.2.2)
+    toxiproxy (2.0.2)
+    tzinfo (2.0.6)
       concurrent-ruby (~> 1.0)
 
 PLATFORMS
+  arm64-darwin-21
+  arm64-darwin-22
   x86_64-darwin-20
   x86_64-linux
 
 DEPENDENCIES
-  activerecord (= 7.0.0.alpha2)
+  activerecord (= 7.0.8)
   after_do
   appraisal
   byebug

data/gemfiles/{activerecord_5.2.gemfile → activerecord_7.1.0.beta1.gemfile}

@@ -2,8 +2,6 @@
 
 source "https://rubygems.org"
 
-gem "activerecord", "5.2.6"
-gem "simplecov", "0.18.5"
-gem "docile", "1.3.5"
+gem "activerecord", "7.1.0.beta1"
 
 gemspec path: "../"

data/gemfiles/activerecord_7.1.0.beta1.gemfile.lock

@@ -0,0 +1,81 @@
+PATH
+  remote: ..
+  specs:
+    lhm-shopify (4.0.0)
+      retriable (>= 3.0.0)
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    activemodel (7.1.0.beta1)
+      activesupport (= 7.1.0.beta1)
+    activerecord (7.1.0.beta1)
+      activemodel (= 7.1.0.beta1)
+      activesupport (= 7.1.0.beta1)
+      timeout (>= 0.4.0)
+    activesupport (7.1.0.beta1)
+      base64
+      bigdecimal
+      concurrent-ruby (~> 1.0, >= 1.0.2)
+      connection_pool (>= 2.2.5)
+      drb
+      i18n (>= 1.6, < 2)
+      minitest (>= 5.1)
+      mutex_m
+      tzinfo (~> 2.0)
+    after_do (0.4.0)
+    appraisal (2.5.0)
+      bundler
+      rake
+      thor (>= 0.14.0)
+    base64 (0.1.1)
+    bigdecimal (3.1.4)
+    byebug (11.1.3)
+    concurrent-ruby (1.2.2)
+    connection_pool (2.4.1)
+    docile (1.4.0)
+    drb (2.1.1)
+      ruby2_keywords
+    i18n (1.14.1)
+      concurrent-ruby (~> 1.0)
+    minitest (5.20.0)
+    mocha (2.1.0)
+      ruby2_keywords (>= 0.0.5)
+    mutex_m (0.1.2)
+    mysql2 (0.5.5)
+    rake (13.0.6)
+    retriable (3.1.2)
+    ruby2_keywords (0.0.5)
+    simplecov (0.22.0)
+      docile (~> 1.1)
+      simplecov-html (~> 0.11)
+      simplecov_json_formatter (~> 0.1)
+    simplecov-html (0.12.3)
+    simplecov_json_formatter (0.1.4)
+    thor (1.2.2)
+    timeout (0.4.0)
+    toxiproxy (2.0.2)
+    tzinfo (2.0.6)
+      concurrent-ruby (~> 1.0)
+
+PLATFORMS
+  arm64-darwin-21
+  arm64-darwin-22
+  x86_64-darwin-20
+  x86_64-linux
+
+DEPENDENCIES
+  activerecord (= 7.1.0.beta1)
+  after_do
+  appraisal
+  byebug
+  lhm-shopify!
+  minitest
+  mocha
+  mysql2
+  rake
+  simplecov
+  toxiproxy
+
+BUNDLED WITH
+   2.2.22

data/lhm.gemspec

@@ -21,7 +21,7 @@ Gem::Specification.new do |s|
   s.executables   = []
   s.metadata['allowed_push_host'] = "https://rubygems.org"
 
-  s.required_ruby_version = '>= 2.3.0'
+  s.required_ruby_version = '>= 3.0.0'
 
   s.add_dependency 'retriable', '>= 3.0.0'
 

data/lib/lhm/sql_helper.rb

@@ -34,7 +34,7 @@ module Lhm
 
     def column_definition(cols)
       Array(cols).map do |column|
-        column.to_s.match(/`?([^\(]+)`?(\([^\)]+\))?/).captures
+        column.to_s.match(/`?([^\(\s]+)`?\s*(\([^\)]+\))?/).captures
       end
     end