lhm-shopify 3.5.5 → 4.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: c3893b743c675e62933e58a4f56c0ac2dba1b1081afabc779dc1d899406e7c42
-  data.tar.gz: 569175938b8069e0036f881400b9427180283ec61c29ca5d509a7090a7f8a9ba
+  metadata.gz: 850cda6d2f610db2985c780d2a90d176e8520b373e21dafbe67df47e9a07ba0f
+  data.tar.gz: 982ab91c6319f5f7803d73d9f3734a57d28a13e76184500a665a5ab5b91d434f
 SHA512:
-  metadata.gz: ba3ebf953f97cd793c52cb83fee77dd4436877ca01a51ee6d718c7937916e391a32c659b9f403b62f65cf0a326dc755e6221e3311760738c9c6fb442746e1617
-  data.tar.gz: 2e617329a895d00a2541539526e941fc94dfda8285083d46d4179587a31786cbbb1062bd8dbdfffcc4d046801944ac96aa700c32618a4db9f4346345e9105c7c
+  metadata.gz: 8bdb254a7cd2091f7d1d125080bc843b43fc9039a7b2a99cba62752bee84726971e8018fb77738fdd0b15867a7f959ff419802270de75c08c2a72cd44d8b004b
+  data.tar.gz: b7d3a989384483ba9f43dd000f1e46c62805e15a0e15c4fcba0fedc964787d4fb1eafe294a2b23deae2d5b035057739551385eca08d24529d8c8e951ba44b163

data/.github/workflows/test.yml

@@ -14,30 +14,32 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        include:
-          - ruby-version: "2.4"
-            activerecord-version: "5.2"
-          - ruby-version: "2.5"
-            activerecord-version: "6.0"
-          - ruby-version: "2.7.5"
-            activerecord-version: "6.1"
-          - ruby-version: "3.0"
-            activerecord-version: "7.0.0.alpha2"
+        activerecord: ["6.1", "7.0", "7.1"]
+        ruby: ["3.0", "3.1", "3.2", "head"]
+        mysql: ["5.7", "8.0"]
+        adapter: ["mysql2", "trilogy"]
+
+    env:
+      BUNDLE_GEMFILE: "${{ github.workspace }}/gemfiles/activerecord_${{ matrix.activerecord }}.gemfile"
+      DATABASE_ADAPTER: "${{ matrix.adapter }}"
+
     steps:
-      - uses: actions/checkout@v2
-      - name: Set up Ruby
-        uses: ruby/setup-ruby@v1
+      - uses: actions/checkout@v4
+      - uses: ruby/setup-ruby@v1
         with:
-          ruby-version: ${{matrix.ruby-version}}
-          bundler-cache: false
-      - name: Install Ruby packages
-        run: BUNDLE_GEMFILE="${GITHUB_WORKSPACE}/gemfiles/activerecord_${{ matrix.activerecord-version }}.gemfile" bundle install
+          ruby-version: ${{matrix.ruby}}
+          bundler-cache: true
+
       - name: Install Ubuntu packages
         run: sudo apt-get update && sudo apt-get install numactl libaio-dev libmysqlclient-dev
+
       - name: Setup MySQL and ProxySQL (docker-compose)
-        run: docker-compose up -d # Might have to change to docker compose up -d (i.e. Compose V2) when the Ubuntu image changes the docker-compose version
+        # Might have to change to docker compose up -d (i.e. Compose V2) when the Ubuntu image changes the docker-compose version
+        run: docker-compose -f docker-compose-mysql-${{ matrix.mysql }}.yml up -d
+
       - name: Wait until DBs are alive
         run: ./scripts/helpers/wait-for-dbs.sh
         timeout-minutes: 2
+
       - name: Run tests
-        run: BUNDLE_GEMFILE="${GITHUB_WORKSPACE}/gemfiles/activerecord_${{ matrix.activerecord-version }}.gemfile" bundle exec rake specs
+        run: bundle exec rake specs

data/Appraisals

@@ -1,24 +1,13 @@
-# Oldest supported Rails version
-appraise "activerecord-5.2" do
-  gem "activerecord", "5.2.6"
-
-# This gemfile will be ran against ruby 2.4 and some gems might require 2.4+. The following list are the gems
-# and their respective versions that work with ruby <= 2.4
-  gem "simplecov", "0.18.5"
-  gem "docile", "1.3.5"
+appraise "activerecord-6.1" do
+  gem "activerecord", "6.1.0"
+  gem "activerecord-trilogy-adapter"
 end
 
-# First conflicted version
-appraise "activerecord-6.0" do
-  gem "activerecord", "6.0.0"
+appraise "activerecord-7.0" do
+  gem "activerecord", "7.0.8"
+  gem "activerecord-trilogy-adapter"
 end
 
-# Second conflicted version
-appraise "activerecord-6.1" do
-  gem "activerecord", "6.1.0"
+appraise "activerecord-7.1" do
+  gem "activerecord", "7.1.1"
 end
-
-# Latest version at the moment
-appraise "activerecord-7.0.0.alpha2" do
-  gem "activerecord", "7.0.0.alpha2"
-end

data/CHANGELOG.md

@@ -1,3 +1,19 @@
+# Unreleased
+
+# 4.1.0 (Oct, 2023)
+* Test against MySQL 8.0.
+* Test against Ruby Head.
+* Drop support for ActiveRecord below version 6.1, as it has reached EOL.
+* Add support for Trilogy MySQL client. It's works with built in ActiveRecord adapter from Rails 7.1 on, as well as dedicated one in older Rails versions.
+
+# 4.0.0 (Sep, 2023)
+* Deprecate `SlaveLag` throttler class name. Use `ReplicaLag` instead (https://github.com/Shopify/lhm/pull/144)
+* Deprecate `slave_lag_throttler` throttler config value. Use `replica_lag_throttler` instead (https://github.com/Shopify/lhm/pull/144)
+* Fix errors when creating indexes with whitespace between column names and sizes. (https://github.com/Shopify/lhm/pull/145)
+* Test against Ruby 3.2 and Rails 7.1.0.beta1. (https://github.com/Shopify/lhm/pull/146)
+* Drop support for Ruby 2 and Rails 5. (https://github.com/Shopify/lhm/pull/148)
+* Fix thread throttler #stride API. (https://github.com/Shopify/lhm/pull/131)
+
 # 3.5.5 (Jan, 2022)
 * Fix error where from Config shadowing which would cause LHM to abort on reconnect (https://github.com/Shopify/lhm/pull/128)
 

data/Gemfile.lock

@@ -1,51 +1,67 @@
 PATH
   remote: .
   specs:
-    lhm-shopify (3.5.5)
+    lhm-shopify (4.1.0)
       retriable (>= 3.0.0)
 
 GEM
   remote: https://rubygems.org/
   specs:
-    activemodel (6.1.4.1)
-      activesupport (= 6.1.4.1)
-    activerecord (6.1.4.1)
-      activemodel (= 6.1.4.1)
-      activesupport (= 6.1.4.1)
-    activesupport (6.1.4.1)
+    activemodel (7.1.1)
+      activesupport (= 7.1.1)
+    activerecord (7.1.1)
+      activemodel (= 7.1.1)
+      activesupport (= 7.1.1)
+      timeout (>= 0.4.0)
+    activesupport (7.1.1)
+      base64
+      bigdecimal
       concurrent-ruby (~> 1.0, >= 1.0.2)
+      connection_pool (>= 2.2.5)
+      drb
       i18n (>= 1.6, < 2)
       minitest (>= 5.1)
+      mutex_m
       tzinfo (~> 2.0)
-      zeitwerk (~> 2.3)
     after_do (0.4.0)
-    appraisal (2.4.1)
+    appraisal (2.5.0)
       bundler
       rake
       thor (>= 0.14.0)
+    base64 (0.1.1)
+    bigdecimal (3.1.4)
     byebug (11.1.3)
-    concurrent-ruby (1.1.9)
+    concurrent-ruby (1.2.2)
+    connection_pool (2.4.1)
     docile (1.4.0)
-    i18n (1.8.11)
+    drb (2.1.1)
+      ruby2_keywords
+    i18n (1.14.1)
       concurrent-ruby (~> 1.0)
-    minitest (5.14.4)
-    mocha (1.13.0)
-    mysql2 (0.5.3)
+    minitest (5.20.0)
+    mocha (2.1.0)
+      ruby2_keywords (>= 0.0.5)
+    mutex_m (0.1.2)
+    mysql2 (0.5.5)
     rake (13.0.6)
     retriable (3.1.2)
-    simplecov (0.21.2)
+    ruby2_keywords (0.0.5)
+    simplecov (0.22.0)
       docile (~> 1.1)
       simplecov-html (~> 0.11)
       simplecov_json_formatter (~> 0.1)
     simplecov-html (0.12.3)
-    simplecov_json_formatter (0.1.3)
-    thor (1.1.0)
-    toxiproxy (2.0.0)
-    tzinfo (2.0.4)
+    simplecov_json_formatter (0.1.4)
+    thor (1.2.2)
+    timeout (0.4.0)
+    toxiproxy (2.0.2)
+    trilogy (2.6.0)
+    tzinfo (2.0.6)
       concurrent-ruby (~> 1.0)
-    zeitwerk (2.5.1)
 
 PLATFORMS
+  arm64-darwin-21
+  arm64-darwin-22
   x86_64-darwin-20
   x86_64-linux
 
@@ -61,6 +77,7 @@ DEPENDENCIES
   rake
   simplecov
   toxiproxy
+  trilogy
 
 BUNDLED WITH
    2.2.22

data/README.md

@@ -2,7 +2,7 @@
 
 [![Tests](https://github.com/Shopify/lhm/actions/workflows/test.yml/badge.svg)](https://github.com/Shopify/lhm/actions/workflows/test.yml)
 
-This is the Shopify fork of [SoundCloud's LHM](https://github.com/soundcloud/lhm). The 
+This is the Shopify fork of [SoundCloud's LHM](https://github.com/soundcloud/lhm). The
 following description, originally from SoundCloud (with minor updates by Shopify),
 gives some of the flavor around its original creation, and its choice of name...
 
@@ -57,7 +57,7 @@ a single integer numeric key column named `id`.
 
 ## Installation
 
-Install it via `gem install lhm` or by adding `gem "lhm"` to your `Gemfile`.
+Install it via `gem install lhm-shopify` or by adding `gem "lhm-shopify"` to your `Gemfile`.
 
 ## Usage
 
@@ -112,7 +112,7 @@ tables must be cleaned up.
 ### Usage with ProxySQL
 LHM can recover from connection loss. However, when used in conjunction with ProxySQL, there are multiple ways that
 connection loss could induce data loss (if triggered by a failover). Therefore  it will perform additional checks to
-ensure that the MySQL host stays consistent across the schema migrations if the feature is enabled. 
+ensure that the MySQL host stays consistent across the schema migrations if the feature is enabled.
 This is done by tagging every query with `/*maintenance:lhm*/`, which will be recognized by ProxySQL.
 However, to get this feature working, a new ProxySQL query rule must be added.
 ```cnf
@@ -127,7 +127,7 @@ However, to get this feature working, a new ProxySQL query rule must be added.
 This will ensure that all relevant queries are forwarded to the current writer.
 
 Also, ProxySQL disables [multiplexing](https://proxysql.com/documentation/multiplexing/) for `select` on `@@` variables.
-Therefore, the following rules must be added to ensure that queries (even if tagged with `/*maintenance:lhm*/`) get 
+Therefore, the following rules must be added to ensure that queries (even if tagged with `/*maintenance:lhm*/`) get
 forwarded to the right target.
 ```cnf
 {
@@ -144,7 +144,7 @@ forwarded to the right target.
 }
 ```
 
-Once these changes are added to the ProxySQL configuration (either through `.cnf` or dynamically through the admin interface), 
+Once these changes are added to the ProxySQL configuration (either through `.cnf` or dynamically through the admin interface),
 the feature can be enabled. This is done by adding this flag when providing options to the migration:
 ```ruby
  Lhm.change_table(..., options: {reconnect_with_consistent_host: true}) do |t|
@@ -165,13 +165,13 @@ Lhm.change_table :users, throttler: my_throttler  do |m|
 end
 ```
 
-### SlaveLag Throttler
+### ReplicaLag Throttler
 
 Lhm uses by default the time throttler, however a better solution is to throttle the copy of the data
-depending on the time that the slaves are behind. To use the SlaveLag throttler:
+depending on the time that the replicas are behind. To use the ReplicaLag throttler:
 
 ```ruby
-Lhm.change_table :users, throttler: :slave_lag_throttler  do |m|
+Lhm.change_table :users, throttler: :replica_lag_throttler  do |m|
   ...
 end
 ```
@@ -179,7 +179,7 @@ end
 Or to set that as default throttler, use the following (for instance in a Rails initializer):
 
 ```ruby
-Lhm.setup_throttler(:slave_lag_throttler)
+Lhm.setup_throttler(:replica_lag_throttler)
 ```
 
 ### ThreadsRunning Throttler
@@ -189,6 +189,13 @@ throttle based on the number of threads running in MySQL, as a proxy for "is
 this operation causing excessive load":
 
 ```ruby
+my_throttler = Lhm::Throttler::ThreadsRunning.new(stride: 100_000)
+
+Lhm.change_table :users, throttler: my_throttler  do |m|
+  ...
+end
+
+# or use default settings:
 Lhm.change_table :users, throttler: :threads_running_throttler do |m|
   ...
 end
@@ -278,7 +285,7 @@ To run the tests:
 ```bash
 bundle exec rake unit # unit tests
 bundle exec rake integration # integration tests
-bundle exec rake unit # all tests
+bundle exec rake specs # all tests
 ```
 
 You can run an individual test as follows:
@@ -304,15 +311,15 @@ open coverage/index.html
 ```
 
 ### Merging for a new version
-When creating a PR for a new version, make sure that th version has been bumped in `lib/lhm/version.rb`. Then run the following code snippet to ensure the everything is consistent, otherwise 
+When creating a PR for a new version, make sure that th version has been bumped in `lib/lhm/version.rb`. Then run the following code snippet to ensure the everything is consistent, otherwise
 the gem will not publish.
 ```bash
 bundle install
 bundle update
-bundle exec appraisals install
+bundle exec appraisal install
 ```
 
-### Docker Compose
+### Podman Compose
 The integration tests rely on a replication configuration for MySQL which is being proxied by an instance of ProxySQL.
 It is important that every container is running to execute the integration test suite.
 
@@ -327,7 +334,7 @@ The license is included as [LICENSE](LICENSE) in this directory.
   * [Twitter][2]
   * [pt-online-schema-change][3]
 
-[0]: http://openarkkit.googlecode.com
+[0]: https://shlomi-noach.github.io/openarkkit/introduction.html
 [1]: http://www.facebook.com/note.php?note\_id=430801045932
 [2]: https://github.com/freels/table_migrator
 [3]: http://www.percona.com/doc/percona-toolkit/2.1/pt-online-schema-change.html

data/dev.yml

@@ -5,17 +5,17 @@ up:
           or: [mysql@5.7]
           conflicts: [shopify/shopify/mysql-client, mysql-connector-c, mysql, mysql-client]
       - wget
-  - ruby: 2.7.5
+  - ruby: 3.2.2
   - bundler
+  - podman
   - custom:
       name: Get Appraisal gems
       met?: bundle exec appraisal install
       meet: ":"
-  - docker
   - custom:
-      name: Docker Compose
-      met?: docker compose ls | grep -ioE -q "lhm.*running\(4\)"
-      meet: docker compose up -d
+      name: Podman compose
+      met?: podman-compose ps | grep -ioE -q "lhm.*running\(4\)"
+      meet: podman-compose up -d
   - custom:
       name: Waiting for DBs to be operational
       met?: ./scripts/helpers/wait-for-dbs.sh
@@ -25,7 +25,8 @@ commands:
   unit: bundle exec rake unit
   int: bundle exec rake integration
   test:
-    optional: file
+    syntax:
+      optional: file
     aliases: [ t ]
     run: |
       if [[ $# -eq 0 ]]; then
@@ -37,9 +38,12 @@ commands:
   cov: rm -rf coverage; COV=1 bundle exec rake unit && bundle exec rake integration; open coverage/index.html
   logs:
     desc: "See the DB logs (ctrl-c + ctrl-c to exit)"
-    run: docker-compose logs -f
+    run: podman-compose logs -f
   clear:
-    run: docker-compose rm -v -s -f && docker-compose up -d && ./scripts/helpers/wait-for-dbs.sh
+    run: podman-compose down -v && podman-compose up -d && ./scripts/helpers/wait-for-dbs.sh
+    subcommands:
+      mysql-5.7: podman-compose down -v && podman-compose -f docker-compose-mysql-5.7.yml up -d && ./scripts/helpers/wait-for-dbs.sh
+      mysql-8.0: podman-compose down -v && podman-compose -f docker-compose-mysql-8.0.yml up -d && ./scripts/helpers/wait-for-dbs.sh
   pre-publish:
     # Ensures all Gemfile.lock are sync with the new version in `lhm/version.rb` and runs appraisals
     run: bundle install && bundle exec appraisal install && bundle exec appraisal rake specs

data/docker-compose-mysql-5.7.yml

@@ -0,0 +1 @@
+docker-compose.yml

data/docker-compose-mysql-8.0.yml

@@ -0,0 +1,63 @@
+services:
+  #  Writer
+  mysql-1:
+    container_name: mysql-1
+    image: percona:8.0
+    platform: linux/amd64
+    command:
+      --server-id=1
+      --log-bin
+      --log-slave-updates=ON
+      --gtid-mode=ON
+      --enforce-gtid-consistency=ON
+      --read-only=OFF
+      --max-connections=1000
+      --default-authentication-plugin=mysql_native_password
+    hostname: 'mysql-1'
+    volumes:
+      - ./scripts/mysql/writer:/docker-entrypoint-initdb.d
+    environment:
+      MYSQL_ROOT_PASSWORD: password
+      MYSQL_HOST: mysql-1
+    ports:
+      - "13006:3306"
+  #  Reader
+  mysql-2:
+    container_name: mysql-2
+    image: percona:8.0
+    platform: linux/amd64
+    command:
+      --server-id=2
+      --log-bin
+      --log-slave-updates=ON
+      --gtid-mode=ON
+      --enforce-gtid-consistency=ON
+      --read-only=ON
+      --max-connections=1000
+      --default-authentication-plugin=mysql_native_password
+    hostname: 'mysql-2'
+    volumes:
+      - ./scripts/mysql/reader:/docker-entrypoint-initdb.d
+    environment:
+      MYSQL_ROOT_PASSWORD: password
+      MYSQL_HOST: mysql-2
+    ports:
+      - "13007:3306"
+  #  Proxysql
+  proxysql:
+    container_name: proxysql
+    image: proxysql/proxysql:2.0.11
+    platform: linux/amd64
+    volumes:
+      - ./scripts/proxysql/proxysql.cnf:/etc/proxysql.cnf
+    command: "proxysql -c /etc/proxysql.cnf -f --idle-threads"
+    ports:
+      - "13005:3306"
+      - "6032:6032"
+  toxiproxy:
+    container_name: toxiproxy
+    image: "ghcr.io/shopify/toxiproxy"
+    ports:
+      - "8474:8474"
+      - "22220:22220"
+      - "22222:22222"

data/docker-compose.yml

@@ -3,6 +3,7 @@ services:
   mysql-1:
     container_name: mysql-1
     image: percona:5.7
+    platform: linux/amd64
     command:
       --server-id=1
       --log-bin
@@ -18,11 +19,12 @@ services:
       MYSQL_ROOT_PASSWORD: password
       MYSQL_HOST: mysql-1
     ports:
-      - "33006:3306"
+      - "13006:3306"
   #  Reader
   mysql-2:
     container_name: mysql-2
     image: percona:5.7
+    platform: linux/amd64
     command:
       --server-id=2
       --log-bin
@@ -38,7 +40,7 @@ services:
       MYSQL_ROOT_PASSWORD: password
       MYSQL_HOST: mysql-2
     ports:
-      - "33007:3306"
+      - "13007:3306"
   #  Proxysql
   proxysql:
     container_name: proxysql
@@ -47,7 +49,7 @@ services:
       - ./scripts/proxysql/proxysql.cnf:/etc/proxysql.cnf
     command: "proxysql -c /etc/proxysql.cnf -f --idle-threads"
     ports:
-      - "33005:3306"
+      - "13005:3306"
       - "6032:6032"
   toxiproxy:
     container_name: toxiproxy

data/gemfiles/activerecord_6.1.gemfile

@@ -3,5 +3,6 @@
 source "https://rubygems.org"
 
 gem "activerecord", "6.1.0"
+gem "activerecord-trilogy-adapter"
 
 gemspec path: "../"

data/gemfiles/activerecord_6.1.gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: ..
   specs:
-    lhm-shopify (3.5.5)
+    lhm-shopify (4.1.0)
       retriable (>= 3.0.0)
 
 GEM
@@ -12,6 +12,9 @@ GEM
     activerecord (6.1.0)
       activemodel (= 6.1.0)
       activesupport (= 6.1.0)
+    activerecord-trilogy-adapter (3.1.2)
+      activerecord (>= 6.0.a, < 7.1.a)
+      trilogy (>= 2.4.0)
     activesupport (6.1.0)
       concurrent-ruby (~> 1.0, >= 1.0.2)
       i18n (>= 1.6, < 2)
@@ -19,38 +22,44 @@ GEM
       tzinfo (~> 2.0)
       zeitwerk (~> 2.3)
     after_do (0.4.0)
-    appraisal (2.4.1)
+    appraisal (2.5.0)
       bundler
       rake
       thor (>= 0.14.0)
     byebug (11.1.3)
-    concurrent-ruby (1.1.9)
+    concurrent-ruby (1.2.2)
     docile (1.4.0)
-    i18n (1.8.11)
+    i18n (1.14.1)
       concurrent-ruby (~> 1.0)
-    minitest (5.14.4)
-    mocha (1.13.0)
-    mysql2 (0.5.3)
+    minitest (5.20.0)
+    mocha (2.1.0)
+      ruby2_keywords (>= 0.0.5)
+    mysql2 (0.5.5)
     rake (13.0.6)
     retriable (3.1.2)
-    simplecov (0.21.2)
+    ruby2_keywords (0.0.5)
+    simplecov (0.22.0)
       docile (~> 1.1)
       simplecov-html (~> 0.11)
       simplecov_json_formatter (~> 0.1)
     simplecov-html (0.12.3)
-    simplecov_json_formatter (0.1.3)
-    thor (1.1.0)
-    toxiproxy (2.0.0)
-    tzinfo (2.0.4)
+    simplecov_json_formatter (0.1.4)
+    thor (1.2.2)
+    toxiproxy (2.0.2)
+    trilogy (2.6.0)
+    tzinfo (2.0.6)
       concurrent-ruby (~> 1.0)
-    zeitwerk (2.5.1)
+    zeitwerk (2.6.12)
 
 PLATFORMS
+  arm64-darwin-21
+  arm64-darwin-22
   x86_64-darwin-20
   x86_64-linux
 
 DEPENDENCIES
   activerecord (= 6.1.0)
+  activerecord-trilogy-adapter
   after_do
   appraisal
   byebug
@@ -61,6 +70,7 @@ DEPENDENCIES
   rake
   simplecov
   toxiproxy
+  trilogy
 
 BUNDLED WITH
    2.2.22

data/gemfiles/{activerecord_7.0.0.alpha2.gemfile → activerecord_7.0.gemfile}

@@ -2,6 +2,7 @@
 
 source "https://rubygems.org"
 
-gem "activerecord", "7.0.0.alpha2"
+gem "activerecord", "7.0.8"
+gem "activerecord-trilogy-adapter"
 
 gemspec path: "../"