lhm-shopify 4.0.0 → 4.1.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 34c64cb93bda1fe53758ea181451438890d8460e167784dfff3255aba7ccf662
-  data.tar.gz: 14e18340ed78985984b93c8d36328ccb9a29d1318639e190ad98f5ceecdf828c
+  metadata.gz: 49f23bf2b018fc5af679644fb6c9f424f943a0d4dd58d7e2d8366b389ae4270b
+  data.tar.gz: 14e15f4fc464d721b20889a1841c4a78a609bb3c6aee1a68ee5f3f5e3e89d2ec
 SHA512:
-  metadata.gz: 4f83e952632a11d868a75da5d9b2e64b7326090ed0b37f74d221f8b86a1345bef783c5d56ca82691d13067b410cf0926d2e88be0939c9cb925b66f418e4fcb9e
-  data.tar.gz: 6aefea0808ed1ba537d550464492784f9b2cc504858a2524cf7c74bde16123c4be0eb0965c206f8f3cb0c2cceca48bc17651d111e8d9401a7f3c762efbcef7a3
+  metadata.gz: 86a7153f5278bdf8d26ebd0ca668dab3e4861ad5349e1115796a59b72ac7c8b0eb920cbe74f443e9913efd5b8c2861499c98ba37990d970177e273188656d02a
+  data.tar.gz: f394a26a3dcf8e5737748610db039b28e9cde415a5a82f3b2f29f358f93124155ee6be68c8196bf027b807aedf19feb6f7a60d89097384242f31e65c96907fc7

data/.github/workflows/test.yml

@@ -14,30 +14,32 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        include:
-          - ruby-version: "3.0"
-            activerecord-version: "6.0"
-          - ruby-version: "3.1"
-            activerecord-version: "6.1"
-          - ruby-version: "3.2"
-            activerecord-version: "7.0"
-          - ruby-version: "3.2"
-            activerecord-version: "7.1.0.beta1"
+        activerecord: ["6.1", "7.0", "7.1"]
+        ruby: ["3.0", "3.1", "3.2", "head"]
+        mysql: ["5.7", "8.0"]
+        adapter: ["mysql2", "trilogy"]
+
+    env:
+      BUNDLE_GEMFILE: "${{ github.workspace }}/gemfiles/activerecord_${{ matrix.activerecord }}.gemfile"
+      DATABASE_ADAPTER: "${{ matrix.adapter }}"
+
     steps:
-      - uses: actions/checkout@v2
-      - name: Set up Ruby
-        uses: ruby/setup-ruby@v1
+      - uses: actions/checkout@v4
+      - uses: ruby/setup-ruby@v1
         with:
-          ruby-version: ${{matrix.ruby-version}}
-          bundler-cache: false
-      - name: Install Ruby packages
-        run: BUNDLE_GEMFILE="${GITHUB_WORKSPACE}/gemfiles/activerecord_${{ matrix.activerecord-version }}.gemfile" bundle install
+          ruby-version: ${{matrix.ruby}}
+          bundler-cache: true
+
       - name: Install Ubuntu packages
         run: sudo apt-get update && sudo apt-get install numactl libaio-dev libmysqlclient-dev
+
       - name: Setup MySQL and ProxySQL (docker-compose)
-        run: docker-compose up -d # Might have to change to docker compose up -d (i.e. Compose V2) when the Ubuntu image changes the docker-compose version
+        # Might have to change to docker compose up -d (i.e. Compose V2) when the Ubuntu image changes the docker-compose version
+        run: docker-compose -f docker-compose-mysql-${{ matrix.mysql }}.yml up -d
+
       - name: Wait until DBs are alive
         run: ./scripts/helpers/wait-for-dbs.sh
         timeout-minutes: 2
+
       - name: Run tests
-        run: BUNDLE_GEMFILE="${GITHUB_WORKSPACE}/gemfiles/activerecord_${{ matrix.activerecord-version }}.gemfile" bundle exec rake specs
+        run: bundle exec rake specs

data/Appraisals

@@ -1,19 +1,13 @@
-# First conflicted version
-appraise "activerecord-6.0" do
-  gem "activerecord", "6.0.3"
-end
-
-# Second conflicted version
 appraise "activerecord-6.1" do
   gem "activerecord", "6.1.0"
+  gem "activerecord-trilogy-adapter"
 end
 
-# Latest version at the moment
 appraise "activerecord-7.0" do
   gem "activerecord", "7.0.8"
+  gem "activerecord-trilogy-adapter"
 end
 
-# Next release
-appraise "activerecord-7.1.0.beta1" do
-  gem "activerecord", "7.1.0.beta1"
-end
+appraise "activerecord-7.1" do
+  gem "activerecord", "7.1.1"
+end

data/CHANGELOG.md

@@ -1,5 +1,14 @@
 # Unreleased
 
+# 4.1.1 (Nov, 2023)
+* Fix check for warnings against PK in MySQL 8+
+
+# 4.1.0 (Oct, 2023)
+* Test against MySQL 8.0.
+* Test against Ruby Head.
+* Drop support for ActiveRecord below version 6.1, as it has reached EOL.
+* Add support for Trilogy MySQL client. It's works with built in ActiveRecord adapter from Rails 7.1 on, as well as dedicated one in older Rails versions.
+
 # 4.0.0 (Sep, 2023)
 * Deprecate `SlaveLag` throttler class name. Use `ReplicaLag` instead (https://github.com/Shopify/lhm/pull/144)
 * Deprecate `slave_lag_throttler` throttler config value. Use `replica_lag_throttler` instead (https://github.com/Shopify/lhm/pull/144)

data/Gemfile.lock

@@ -1,35 +1,47 @@
 PATH
   remote: .
   specs:
-    lhm-shopify (4.0.0)
+    lhm-shopify (4.1.1)
       retriable (>= 3.0.0)
 
 GEM
   remote: https://rubygems.org/
   specs:
-    activemodel (7.0.8)
-      activesupport (= 7.0.8)
-    activerecord (7.0.8)
-      activemodel (= 7.0.8)
-      activesupport (= 7.0.8)
-    activesupport (7.0.8)
+    activemodel (7.1.1)
+      activesupport (= 7.1.1)
+    activerecord (7.1.1)
+      activemodel (= 7.1.1)
+      activesupport (= 7.1.1)
+      timeout (>= 0.4.0)
+    activesupport (7.1.1)
+      base64
+      bigdecimal
       concurrent-ruby (~> 1.0, >= 1.0.2)
+      connection_pool (>= 2.2.5)
+      drb
       i18n (>= 1.6, < 2)
       minitest (>= 5.1)
+      mutex_m
       tzinfo (~> 2.0)
     after_do (0.4.0)
     appraisal (2.5.0)
       bundler
       rake
       thor (>= 0.14.0)
+    base64 (0.1.1)
+    bigdecimal (3.1.4)
     byebug (11.1.3)
     concurrent-ruby (1.2.2)
+    connection_pool (2.4.1)
     docile (1.4.0)
+    drb (2.1.1)
+      ruby2_keywords
     i18n (1.14.1)
       concurrent-ruby (~> 1.0)
     minitest (5.20.0)
     mocha (2.1.0)
       ruby2_keywords (>= 0.0.5)
+    mutex_m (0.1.2)
     mysql2 (0.5.5)
     rake (13.0.6)
     retriable (3.1.2)
@@ -41,7 +53,9 @@ GEM
     simplecov-html (0.12.3)
     simplecov_json_formatter (0.1.4)
     thor (1.2.2)
+    timeout (0.4.0)
     toxiproxy (2.0.2)
+    trilogy (2.6.0)
     tzinfo (2.0.6)
       concurrent-ruby (~> 1.0)
 
@@ -63,6 +77,7 @@ DEPENDENCIES
   rake
   simplecov
   toxiproxy
+  trilogy
 
 BUNDLED WITH
    2.2.22

data/README.md

@@ -2,7 +2,7 @@
 
 [![Tests](https://github.com/Shopify/lhm/actions/workflows/test.yml/badge.svg)](https://github.com/Shopify/lhm/actions/workflows/test.yml)
 
-This is the Shopify fork of [SoundCloud's LHM](https://github.com/soundcloud/lhm). The 
+This is the Shopify fork of [SoundCloud's LHM](https://github.com/soundcloud/lhm). The
 following description, originally from SoundCloud (with minor updates by Shopify),
 gives some of the flavor around its original creation, and its choice of name...
 
@@ -112,7 +112,7 @@ tables must be cleaned up.
 ### Usage with ProxySQL
 LHM can recover from connection loss. However, when used in conjunction with ProxySQL, there are multiple ways that
 connection loss could induce data loss (if triggered by a failover). Therefore  it will perform additional checks to
-ensure that the MySQL host stays consistent across the schema migrations if the feature is enabled. 
+ensure that the MySQL host stays consistent across the schema migrations if the feature is enabled.
 This is done by tagging every query with `/*maintenance:lhm*/`, which will be recognized by ProxySQL.
 However, to get this feature working, a new ProxySQL query rule must be added.
 ```cnf
@@ -127,7 +127,7 @@ However, to get this feature working, a new ProxySQL query rule must be added.
 This will ensure that all relevant queries are forwarded to the current writer.
 
 Also, ProxySQL disables [multiplexing](https://proxysql.com/documentation/multiplexing/) for `select` on `@@` variables.
-Therefore, the following rules must be added to ensure that queries (even if tagged with `/*maintenance:lhm*/`) get 
+Therefore, the following rules must be added to ensure that queries (even if tagged with `/*maintenance:lhm*/`) get
 forwarded to the right target.
 ```cnf
 {
@@ -144,7 +144,7 @@ forwarded to the right target.
 }
 ```
 
-Once these changes are added to the ProxySQL configuration (either through `.cnf` or dynamically through the admin interface), 
+Once these changes are added to the ProxySQL configuration (either through `.cnf` or dynamically through the admin interface),
 the feature can be enabled. This is done by adding this flag when providing options to the migration:
 ```ruby
  Lhm.change_table(..., options: {reconnect_with_consistent_host: true}) do |t|
@@ -285,7 +285,7 @@ To run the tests:
 ```bash
 bundle exec rake unit # unit tests
 bundle exec rake integration # integration tests
-bundle exec rake unit # all tests
+bundle exec rake specs # all tests
 ```
 
 You can run an individual test as follows:
@@ -311,12 +311,12 @@ open coverage/index.html
 ```
 
 ### Merging for a new version
-When creating a PR for a new version, make sure that th version has been bumped in `lib/lhm/version.rb`. Then run the following code snippet to ensure the everything is consistent, otherwise 
+When creating a PR for a new version, make sure that th version has been bumped in `lib/lhm/version.rb`. Then run the following code snippet to ensure the everything is consistent, otherwise
 the gem will not publish.
 ```bash
 bundle install
 bundle update
-bundle exec appraisals install
+bundle exec appraisal install
 ```
 
 ### Podman Compose

data/dev.yml

@@ -14,7 +14,7 @@ up:
       meet: ":"
   - custom:
       name: Podman compose
-      met?: podman-compose ls | grep -ioE -q "lhm.*running\(4\)"
+      met?: podman-compose ps | grep -ioE -q "lhm.*running\(4\)"
       meet: podman-compose up -d
   - custom:
       name: Waiting for DBs to be operational
@@ -41,6 +41,9 @@ commands:
     run: podman-compose logs -f
   clear:
     run: podman-compose down -v && podman-compose up -d && ./scripts/helpers/wait-for-dbs.sh
+    subcommands:
+      mysql-5.7: podman-compose down -v && podman-compose -f docker-compose-mysql-5.7.yml up -d && ./scripts/helpers/wait-for-dbs.sh
+      mysql-8.0: podman-compose down -v && podman-compose -f docker-compose-mysql-8.0.yml up -d && ./scripts/helpers/wait-for-dbs.sh
   pre-publish:
     # Ensures all Gemfile.lock are sync with the new version in `lhm/version.rb` and runs appraisals
     run: bundle install && bundle exec appraisal install && bundle exec appraisal rake specs

data/docker-compose-mysql-5.7.yml

@@ -0,0 +1 @@
+docker-compose.yml

data/docker-compose-mysql-8.0.yml

@@ -0,0 +1,63 @@
+services:
+  #  Writer
+  mysql-1:
+    container_name: mysql-1
+    image: percona:8.0
+    platform: linux/amd64
+    command:
+      --server-id=1
+      --log-bin
+      --log-slave-updates=ON
+      --gtid-mode=ON
+      --enforce-gtid-consistency=ON
+      --read-only=OFF
+      --max-connections=1000
+      --default-authentication-plugin=mysql_native_password
+    hostname: 'mysql-1'
+    volumes:
+      - ./scripts/mysql/writer:/docker-entrypoint-initdb.d
+    environment:
+      MYSQL_ROOT_PASSWORD: password
+      MYSQL_HOST: mysql-1
+    ports:
+      - "13006:3306"
+  #  Reader
+  mysql-2:
+    container_name: mysql-2
+    image: percona:8.0
+    platform: linux/amd64
+    command:
+      --server-id=2
+      --log-bin
+      --log-slave-updates=ON
+      --gtid-mode=ON
+      --enforce-gtid-consistency=ON
+      --read-only=ON
+      --max-connections=1000
+      --default-authentication-plugin=mysql_native_password
+    hostname: 'mysql-2'
+    volumes:
+      - ./scripts/mysql/reader:/docker-entrypoint-initdb.d
+    environment:
+      MYSQL_ROOT_PASSWORD: password
+      MYSQL_HOST: mysql-2
+    ports:
+      - "13007:3306"
+  #  Proxysql
+  proxysql:
+    container_name: proxysql
+    image: proxysql/proxysql:2.0.11
+    platform: linux/amd64
+    volumes:
+      - ./scripts/proxysql/proxysql.cnf:/etc/proxysql.cnf
+    command: "proxysql -c /etc/proxysql.cnf -f --idle-threads"
+    ports:
+      - "13005:3306"
+      - "6032:6032"
+  toxiproxy:
+    container_name: toxiproxy
+    image: "ghcr.io/shopify/toxiproxy"
+    ports:
+      - "8474:8474"
+      - "22220:22220"
+      - "22222:22222"

data/docker-compose.yml

@@ -19,7 +19,7 @@ services:
       MYSQL_ROOT_PASSWORD: password
       MYSQL_HOST: mysql-1
     ports:
-      - "33006:3306"
+      - "13006:3306"
   #  Reader
   mysql-2:
     container_name: mysql-2
@@ -40,7 +40,7 @@ services:
       MYSQL_ROOT_PASSWORD: password
       MYSQL_HOST: mysql-2
     ports:
-      - "33007:3306"
+      - "13007:3306"
   #  Proxysql
   proxysql:
     container_name: proxysql
@@ -49,7 +49,7 @@ services:
       - ./scripts/proxysql/proxysql.cnf:/etc/proxysql.cnf
     command: "proxysql -c /etc/proxysql.cnf -f --idle-threads"
     ports:
-      - "33005:3306"
+      - "13005:3306"
       - "6032:6032"
   toxiproxy:
     container_name: toxiproxy

data/gemfiles/activerecord_6.1.gemfile

@@ -3,5 +3,6 @@
 source "https://rubygems.org"
 
 gem "activerecord", "6.1.0"
+gem "activerecord-trilogy-adapter"
 
 gemspec path: "../"

data/gemfiles/activerecord_6.1.gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: ..
   specs:
-    lhm-shopify (4.0.0)
+    lhm-shopify (4.1.1)
       retriable (>= 3.0.0)
 
 GEM
@@ -12,6 +12,9 @@ GEM
     activerecord (6.1.0)
       activemodel (= 6.1.0)
       activesupport (= 6.1.0)
+    activerecord-trilogy-adapter (3.1.2)
+      activerecord (>= 6.0.a, < 7.1.a)
+      trilogy (>= 2.4.0)
     activesupport (6.1.0)
       concurrent-ruby (~> 1.0, >= 1.0.2)
       i18n (>= 1.6, < 2)
@@ -43,9 +46,10 @@ GEM
     simplecov_json_formatter (0.1.4)
     thor (1.2.2)
     toxiproxy (2.0.2)
+    trilogy (2.6.0)
     tzinfo (2.0.6)
       concurrent-ruby (~> 1.0)
-    zeitwerk (2.6.11)
+    zeitwerk (2.6.12)
 
 PLATFORMS
   arm64-darwin-21
@@ -55,6 +59,7 @@ PLATFORMS
 
 DEPENDENCIES
   activerecord (= 6.1.0)
+  activerecord-trilogy-adapter
   after_do
   appraisal
   byebug
@@ -65,6 +70,7 @@ DEPENDENCIES
   rake
   simplecov
   toxiproxy
+  trilogy
 
 BUNDLED WITH
    2.2.22

data/gemfiles/activerecord_7.0.gemfile

@@ -3,5 +3,6 @@
 source "https://rubygems.org"
 
 gem "activerecord", "7.0.8"
+gem "activerecord-trilogy-adapter"
 
 gemspec path: "../"

data/gemfiles/activerecord_7.0.gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: ..
   specs:
-    lhm-shopify (4.0.0)
+    lhm-shopify (4.1.1)
       retriable (>= 3.0.0)
 
 GEM
@@ -12,6 +12,9 @@ GEM
     activerecord (7.0.8)
       activemodel (= 7.0.8)
       activesupport (= 7.0.8)
+    activerecord-trilogy-adapter (3.1.2)
+      activerecord (>= 6.0.a, < 7.1.a)
+      trilogy (>= 2.4.0)
     activesupport (7.0.8)
       concurrent-ruby (~> 1.0, >= 1.0.2)
       i18n (>= 1.6, < 2)
@@ -42,6 +45,7 @@ GEM
     simplecov_json_formatter (0.1.4)
     thor (1.2.2)
     toxiproxy (2.0.2)
+    trilogy (2.6.0)
     tzinfo (2.0.6)
       concurrent-ruby (~> 1.0)
 
@@ -53,6 +57,7 @@ PLATFORMS
 
 DEPENDENCIES
   activerecord (= 7.0.8)
+  activerecord-trilogy-adapter
   after_do
   appraisal
   byebug
@@ -63,6 +68,7 @@ DEPENDENCIES
   rake
   simplecov
   toxiproxy
+  trilogy
 
 BUNDLED WITH
    2.2.22

data/gemfiles/{activerecord_6.0.gemfile → activerecord_7.1.gemfile}

@@ -2,6 +2,6 @@
 
 source "https://rubygems.org"
 
-gem "activerecord", "6.0.3"
+gem "activerecord", "7.1.1"
 
 gemspec path: "../"

data/gemfiles/{activerecord_7.1.0.beta1.gemfile.lock → activerecord_7.1.gemfile.lock}

@@ -1,19 +1,19 @@
 PATH
   remote: ..
   specs:
-    lhm-shopify (4.0.0)
+    lhm-shopify (4.1.1)
       retriable (>= 3.0.0)
 
 GEM
   remote: https://rubygems.org/
   specs:
-    activemodel (7.1.0.beta1)
-      activesupport (= 7.1.0.beta1)
-    activerecord (7.1.0.beta1)
-      activemodel (= 7.1.0.beta1)
-      activesupport (= 7.1.0.beta1)
+    activemodel (7.1.1)
+      activesupport (= 7.1.1)
+    activerecord (7.1.1)
+      activemodel (= 7.1.1)
+      activesupport (= 7.1.1)
       timeout (>= 0.4.0)
-    activesupport (7.1.0.beta1)
+    activesupport (7.1.1)
       base64
       bigdecimal
       concurrent-ruby (~> 1.0, >= 1.0.2)
@@ -55,6 +55,7 @@ GEM
     thor (1.2.2)
     timeout (0.4.0)
     toxiproxy (2.0.2)
+    trilogy (2.6.0)
     tzinfo (2.0.6)
       concurrent-ruby (~> 1.0)
 
@@ -65,7 +66,7 @@ PLATFORMS
   x86_64-linux
 
 DEPENDENCIES
-  activerecord (= 7.1.0.beta1)
+  activerecord (= 7.1.1)
   after_do
   appraisal
   byebug
@@ -76,6 +77,7 @@ DEPENDENCIES
   rake
   simplecov
   toxiproxy
+  trilogy
 
 BUNDLED WITH
    2.2.22

data/lhm.gemspec

@@ -31,6 +31,7 @@ Gem::Specification.new do |s|
   s.add_development_dependency 'after_do'
   s.add_development_dependency 'rake'
   s.add_development_dependency 'mysql2'
+  s.add_development_dependency 'trilogy'
   s.add_development_dependency 'simplecov'
   s.add_development_dependency 'toxiproxy'
   s.add_development_dependency 'appraisal'

data/lib/lhm/atomic_switcher.rb

@@ -26,14 +26,14 @@ module Lhm
     end
 
     def atomic_switch
-      "rename table `#{ @origin.name }` to `#{ @migration.archive_name }`, " \
-      "`#{ @destination.name }` to `#{ @origin.name }`"
+      "RENAME TABLE `#{ @origin.name }` TO `#{ @migration.archive_name }`, " \
+      "`#{ @destination.name }` TO `#{ @origin.name }`"
     end
 
     def validate
       unless @connection.data_source_exists?(@origin.name) &&
         @connection.data_source_exists?(@destination.name)
-        error "`#{ @origin.name }` and `#{ @destination.name }` must exist"
+        error "`#{ @origin.name }` AND `#{ @destination.name }` MUST EXIST"
       end
     end
 

data/lib/lhm/chunker.rb

@@ -23,6 +23,7 @@ module Lhm
       @chunk_finder = ChunkFinder.new(migration, connection, options)
       @options = options
       @raise_on_warnings = options.fetch(:raise_on_warnings, false)
+      @pk_duplicate_warning_regexp ||= /Duplicate entry .+ for key '(#{@migration.destination_name}\.)?PRIMARY'/
       @verifier = options[:verifier]
       if @throttler = options[:throttler]
         @throttler.connection = @connection if @throttler.respond_to?(:connection=)
@@ -79,12 +80,12 @@ module Lhm
     private
 
     def raise_on_non_pk_duplicate_warning
-      @connection.execute("show warnings", should_retry: true, log_prefix: LOG_PREFIX).each do |level, code, message|
-        unless message.match?(/Duplicate entry .+ for key 'PRIMARY'/)
-          m = "Unexpected warning found for inserted row: #{message}"
-          Lhm.logger.warn(m)
-          raise Error.new(m) if @raise_on_warnings
-        end
+      @connection.select_all("SHOW WARNINGS", should_retry: true, log_prefix: LOG_PREFIX).each do |row|
+        next if row["Message"].match?(@pk_duplicate_warning_regexp)
+
+        m = "Unexpected warning found for inserted row: #{row["Message"]}"
+        Lhm.logger.warn(m)
+        raise Error.new(m) if @raise_on_warnings
       end
     end
 
@@ -100,7 +101,7 @@ module Lhm
     end
 
     def upper_id(next_id, stride)
-      sql = "select id from `#{ @migration.origin_name }` where id >= #{ next_id } order by id limit 1 offset #{ stride - 1}"
+      sql = "SELECT id FROM `#{ @migration.origin_name }` WHERE id >= #{ next_id } ORDER BY id LIMIT 1 OFFSET #{ stride - 1}"
       top = @connection.select_value(sql, should_retry: true, log_prefix: LOG_PREFIX)
 
       [top ? top.to_i : @limit, @limit].min

data/lib/lhm/connection.rb

@@ -75,6 +75,14 @@ module Lhm
       end
     end
 
+    def select_all(query, should_retry: false, log_prefix: nil)
+      if should_retry
+        exec_with_retries(:select_all, query, log_prefix)
+      else
+        exec(:select_all, query)
+      end
+    end
+
     private
 
     def exec(method, sql)
@@ -105,4 +113,4 @@ module Lhm
       lhm_stack.at(first_candidate_index)
     end
   end
-end
+end