lhm-shopify 3.5.0 → 3.5.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 6028b7c6d8d1f2e2d8b786ae3e8f41c4525c7507529085c3427a8dc9cf11bd2e
-  data.tar.gz: de625eda2736c56cfa813ba73f8447f6e798c30080274d91164b895879a7f3a7
+  metadata.gz: 8b5fbf9905c14cd938075e3f6b930de0b52cdbc7c048f8ed572c654824681d99
+  data.tar.gz: c950dd9650c4e04412a91a9d21b6bbff3b3f092ecbbce1474a77518716737d62
 SHA512:
-  metadata.gz: 6561aadaa26f6f28f4fd04987a7fca180e9130194cd756cc5a53a654e060b7bd93efaec78a6b067d39e2b31fd6b4f1826a4de9ac90da162af912b1dd101b8732
-  data.tar.gz: b221afa7215cf36f66fdd48ffd449a2d6034c8710c36d9ccdbf909b2078b83fc610fa1c1170214752371e50f41ed6504186975c80e5a8164ef6b25dce631fe89
+  metadata.gz: e92c7cc7cefa85e18a9c5b3d17966574324c8fe8e067f4ac4203241d18e8dae836c82554f92685e90978af26451422852c654fed8f0b855032dee0ac3d0f130d
+  data.tar.gz: 8755a2814d7d92d015d35ae3636993e39a66b5d27d1980a002eec96a2e33012350310d5f903b5f3b225d6b4bf60295841ee4e35919b927914a0cc700683d357b

data/.github/workflows/test.yml

@@ -10,15 +10,28 @@ jobs:
   test:
     name: Run tests
     runs-on: ubuntu-latest
+
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - ruby-version: "2.4"
+            activerecord-version: "5.2"
+          - ruby-version: "2.5"
+            activerecord-version: "6.0"
+          - ruby-version: "2.7.5"
+            activerecord-version: "6.1"
+          - ruby-version: "3.0"
+            activerecord-version: "7.0.0.alpha2"
     steps:
       - uses: actions/checkout@v2
       - name: Set up Ruby
         uses: ruby/setup-ruby@v1
         with:
-          ruby-version: 2.5.1
-          bundler-cache: true
+          ruby-version: ${{matrix.ruby-version}}
+          bundler-cache: false
       - name: Install Ruby packages
-        run: bundle install
+        run: BUNDLE_GEMFILE="${GITHUB_WORKSPACE}/gemfiles/activerecord_${{ matrix.activerecord-version }}.gemfile" bundle install
       - name: Install Ubuntu packages
         run: sudo apt-get update && sudo apt-get install numactl libaio-dev libmysqlclient-dev
       - name: Setup MySQL and ProxySQL (docker-compose)
@@ -27,4 +40,4 @@ jobs:
         run: ./scripts/helpers/wait-for-dbs.sh
         timeout-minutes: 2
       - name: Run tests
-        run: bundle exec rake specs
+        run: BUNDLE_GEMFILE="${GITHUB_WORKSPACE}/gemfiles/activerecord_${{ matrix.activerecord-version }}.gemfile" bundle exec rake specs

data/.gitignore

@@ -1,6 +1,5 @@
 *.gem
 .bundle
-gemfiles/*.lock
 pkg/*
 .rvmrc
 .ruby-version
@@ -11,4 +10,3 @@ omg.ponies
 *~
 coverage
 .idea/
-Gemfile.lock

data/Appraisals

@@ -0,0 +1,24 @@
+# Oldest supported Rails version
+appraise "activerecord-5.2" do
+  gem "activerecord", "5.2.6"
+
+# This gemfile will be ran against ruby 2.4 and some gems might require 2.4+. The following list are the gems
+# and their respective versions that work with ruby <= 2.4
+  gem "simplecov", "0.18.5"
+  gem "docile", "1.3.5"
+end
+
+# First conflicted version
+appraise "activerecord-6.0" do
+  gem "activerecord", "6.0.0"
+end
+
+# Second conflicted version
+appraise "activerecord-6.1" do
+  gem "activerecord", "6.1.0"
+end
+
+# Latest version at the moment
+appraise "activerecord-7.0.0.alpha2" do
+  gem "activerecord", "7.0.0.alpha2"
+end

data/Gemfile.lock

@@ -0,0 +1,66 @@
+PATH
+  remote: .
+  specs:
+    lhm-shopify (3.5.1)
+      retriable (>= 3.0.0)
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    activemodel (6.1.4.1)
+      activesupport (= 6.1.4.1)
+    activerecord (6.1.4.1)
+      activemodel (= 6.1.4.1)
+      activesupport (= 6.1.4.1)
+    activesupport (6.1.4.1)
+      concurrent-ruby (~> 1.0, >= 1.0.2)
+      i18n (>= 1.6, < 2)
+      minitest (>= 5.1)
+      tzinfo (~> 2.0)
+      zeitwerk (~> 2.3)
+    after_do (0.4.0)
+    appraisal (2.4.1)
+      bundler
+      rake
+      thor (>= 0.14.0)
+    byebug (11.1.3)
+    concurrent-ruby (1.1.9)
+    docile (1.4.0)
+    i18n (1.8.11)
+      concurrent-ruby (~> 1.0)
+    minitest (5.14.4)
+    mocha (1.13.0)
+    mysql2 (0.5.3)
+    rake (13.0.6)
+    retriable (3.1.2)
+    simplecov (0.21.2)
+      docile (~> 1.1)
+      simplecov-html (~> 0.11)
+      simplecov_json_formatter (~> 0.1)
+    simplecov-html (0.12.3)
+    simplecov_json_formatter (0.1.3)
+    thor (1.1.0)
+    toxiproxy (2.0.0)
+    tzinfo (2.0.4)
+      concurrent-ruby (~> 1.0)
+    zeitwerk (2.5.1)
+
+PLATFORMS
+  x86_64-darwin-20
+  x86_64-linux
+
+DEPENDENCIES
+  activerecord
+  after_do
+  appraisal
+  byebug
+  lhm-shopify!
+  minitest
+  mocha
+  mysql2
+  rake
+  simplecov
+  toxiproxy
+
+BUNDLED WITH
+   2.2.22

data/README.md

@@ -109,6 +109,48 @@ end
 to prevent accidental data loss. After successful or failed LHM migrations, these leftover
 tables must be cleaned up.
 
+### Usage with ProxySQL
+LHM can recover from connection loss. However, when used in conjunction with ProxySQL, there are multiple ways that
+connection loss could induce data loss (if triggered by a failover). Therefore  it will perform additional checks to
+ensure that the MySQL host stays consistent across the schema migrations if the feature is enabled. 
+This is done by tagging every query with `/*maintenance:lhm*/`, which will be recognized by ProxySQL. 
+However, to get this feature working, a new ProxySQL query rule must be added.
+```cnf
+{
+  rule_id = <rule id>
+  active = 1
+  match_pattern = "maintenance:lhm"
+  destination_hostgroup = <MySQL writer's hostgroup>
+}
+```
+
+This will ensure that all relevant queries are forwarded to the current writer.
+
+Also, ProxySQL disables [multiplexing](https://proxysql.com/documentation/multiplexing/) for `select` on `@@` variables.
+Therefore, the following rules must be added to ensure that queries (even if tagged with `/*maintenance:lhm*/`) get 
+forwarded to the right target.
+```cnf
+{
+  rule_id = <rule id>
+  active = 1
+  match_digest = "@@global\.server_id"
+  multiplex = 2
+},
+{
+  rule_id = <rule id>
+  active = 1
+  match_digest = "@@global\.hostname"
+  multiplex = 2
+}
+```
+
+Once these changes are added to the ProxySQL configuration (either through `.cnf` or dynamically through the admin interface), 
+the feature can be enabled. This is done by adding this flag when doing the initial setup:
+```ruby
+ Lhm.setup(connection, options: {reconnect_with_consistent_host: true})
+```
+**Note**: This feature is disabled by default
+
 ## Throttler
 
 LHM uses a throttling mechanism to read data in your original table. By default, 2,000 rows are read each 0.1 second. If you want to change that behaviour, you can pass an instance of a throttler with the `throttler` option. In this example, 1,000 rows will be read with a 10 second delay between each processing:

data/Rakefile

@@ -23,6 +23,7 @@ Rake::TestTask.new('dev') do |t|
   t.libs << 'spec'
   t.test_files = FileList[
     'spec/test_helper.rb',
+    'spec/integration/lhm_spec.rb'
   #  Add file to test individually
   ]
   t.verbose = true

data/dev.yml

@@ -5,12 +5,16 @@ up:
           or: [mysql@5.7]
           conflicts: [shopify/shopify/mysql-client, mysql-connector-c, mysql, mysql-client]
       - wget
-  - ruby: 2.6.5
+  - ruby: 2.7.0
   - bundler
+  - custom:
+      name: Get Appraisal gems
+      met?: bundle exec appraisal install
+      meet: ":"
   - docker
   - custom:
       name: Docker Compose
-      met?: docker compose ls | grep -ioE -q "lhm.*running\(3\)"
+      met?: docker compose ls | grep -ioE -q "lhm.*running\(4\)"
       meet: docker compose up -d
   - custom:
       name: Waiting for DBs to be operational
@@ -20,8 +24,19 @@ up:
 commands:
   unit: bundle exec rake unit
   int: bundle exec rake integration
-  test: bundle exec rake unit && bundle exec rake integration
+  test:
+    optional: file
+    aliases: [ t ]
+    run: |
+      if [[ $# -eq 0 ]]; then
+        bundle exec rake unit && bundle exec rake integration
+      else
+        bundle exec rake dev TEST="$@"
+      fi
+  appraisals: bundle exec appraisal rake specs
   cov: rm -rf coverage; COV=1 bundle exec rake unit && bundle exec rake integration; open coverage/index.html
   logs:
     desc: "See the DB logs (ctrl-c + ctrl-c to exit)"
     run: docker-compose logs -f
+  clear:
+    run: docker-compose rm -v -s -f && docker-compose up -d && ./scripts/helpers/wait-for-dbs.sh

data/docker-compose.yml

@@ -1,14 +1,16 @@
 services:
-#  Writer
+  #  Writer
   mysql-1:
     container_name: mysql-1
     image: percona:5.7
     command:
       --server-id=1
+      --log-bin
       --log-slave-updates=ON
       --gtid-mode=ON
       --enforce-gtid-consistency=ON
       --read-only=OFF
+      --max-connections=1000
     hostname: 'mysql-1'
     volumes:
       - ./scripts/mysql/writer:/docker-entrypoint-initdb.d
@@ -17,16 +19,18 @@ services:
       MYSQL_HOST: mysql-1
     ports:
       - "33006:3306"
-#  Reader
+  #  Reader
   mysql-2:
     container_name: mysql-2
     image: percona:5.7
     command:
       --server-id=2
+      --log-bin
       --log-slave-updates=ON
       --gtid-mode=ON
       --enforce-gtid-consistency=ON
       --read-only=ON
+      --max-connections=1000
     hostname: 'mysql-2'
     volumes:
       - ./scripts/mysql/reader:/docker-entrypoint-initdb.d
@@ -35,7 +39,7 @@ services:
       MYSQL_HOST: mysql-2
     ports:
       - "33007:3306"
-#  Proxysql
+  #  Proxysql
   proxysql:
     container_name: proxysql
     image: proxysql/proxysql:2.0.11
@@ -44,3 +48,11 @@ services:
     command: "proxysql -c /etc/proxysql.cnf -f --idle-threads"
     ports:
       - "33005:3306"
+      - "6032:6032"
+  toxiproxy:
+    container_name: toxiproxy
+    image: "ghcr.io/shopify/toxiproxy"
+    ports:
+      - "8474:8474"
+      - "22220:22220"
+      - "22222:22222"

data/gemfiles/activerecord_5.2.gemfile

@@ -0,0 +1,9 @@
+# This file was generated by Appraisal
+
+source "https://rubygems.org"
+
+gem "activerecord", "5.2.6"
+gem "simplecov", "0.18.5"
+gem "docile", "1.3.5"
+
+gemspec path: "../"

data/gemfiles/activerecord_5.2.gemfile.lock

@@ -0,0 +1,65 @@
+PATH
+  remote: ..
+  specs:
+    lhm-shopify (3.5.1)
+      retriable (>= 3.0.0)
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    activemodel (5.2.6)
+      activesupport (= 5.2.6)
+    activerecord (5.2.6)
+      activemodel (= 5.2.6)
+      activesupport (= 5.2.6)
+      arel (>= 9.0)
+    activesupport (5.2.6)
+      concurrent-ruby (~> 1.0, >= 1.0.2)
+      i18n (>= 0.7, < 2)
+      minitest (~> 5.1)
+      tzinfo (~> 1.1)
+    after_do (0.4.0)
+    appraisal (2.4.1)
+      bundler
+      rake
+      thor (>= 0.14.0)
+    arel (9.0.0)
+    byebug (11.1.3)
+    concurrent-ruby (1.1.9)
+    docile (1.3.5)
+    i18n (1.8.11)
+      concurrent-ruby (~> 1.0)
+    minitest (5.14.4)
+    mocha (1.13.0)
+    mysql2 (0.5.3)
+    rake (13.0.6)
+    retriable (3.1.2)
+    simplecov (0.18.5)
+      docile (~> 1.1)
+      simplecov-html (~> 0.11)
+    simplecov-html (0.12.3)
+    thor (1.1.0)
+    thread_safe (0.3.6)
+    toxiproxy (2.0.0)
+    tzinfo (1.2.9)
+      thread_safe (~> 0.1)
+
+PLATFORMS
+  x86_64-darwin-20
+
+DEPENDENCIES
+  activerecord (= 5.2.6)
+  after_do
+  appraisal
+  byebug
+  docile (= 1.3.5)
+  lhm-shopify!
+  minitest
+  mocha
+  mysql2
+  rake
+  simplecov (= 0.18.5)
+  toxiproxy
+
+BUNDLED WITH
+   2.2.22

data/gemfiles/activerecord_6.0.gemfile

@@ -0,0 +1,7 @@
+# This file was generated by Appraisal
+
+source "https://rubygems.org"
+
+gem "activerecord", "6.0.0"
+
+gemspec path: "../"

data/gemfiles/activerecord_6.0.gemfile.lock

@@ -0,0 +1,67 @@
+PATH
+  remote: ..
+  specs:
+    lhm-shopify (3.5.1)
+      retriable (>= 3.0.0)
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    activemodel (6.0.0)
+      activesupport (= 6.0.0)
+    activerecord (6.0.0)
+      activemodel (= 6.0.0)
+      activesupport (= 6.0.0)
+    activesupport (6.0.0)
+      concurrent-ruby (~> 1.0, >= 1.0.2)
+      i18n (>= 0.7, < 2)
+      minitest (~> 5.1)
+      tzinfo (~> 1.1)
+      zeitwerk (~> 2.1, >= 2.1.8)
+    after_do (0.4.0)
+    appraisal (2.4.1)
+      bundler
+      rake
+      thor (>= 0.14.0)
+    byebug (11.1.3)
+    concurrent-ruby (1.1.9)
+    docile (1.4.0)
+    i18n (1.8.11)
+      concurrent-ruby (~> 1.0)
+    minitest (5.14.4)
+    mocha (1.13.0)
+    mysql2 (0.5.3)
+    rake (13.0.6)
+    retriable (3.1.2)
+    simplecov (0.21.2)
+      docile (~> 1.1)
+      simplecov-html (~> 0.11)
+      simplecov_json_formatter (~> 0.1)
+    simplecov-html (0.12.3)
+    simplecov_json_formatter (0.1.3)
+    thor (1.1.0)
+    thread_safe (0.3.6)
+    toxiproxy (2.0.0)
+    tzinfo (1.2.9)
+      thread_safe (~> 0.1)
+    zeitwerk (2.5.1)
+
+PLATFORMS
+  x86_64-darwin-20
+  x86_64-linux
+
+DEPENDENCIES
+  activerecord (= 6.0.0)
+  after_do
+  appraisal
+  byebug
+  lhm-shopify!
+  minitest
+  mocha
+  mysql2
+  rake
+  simplecov
+  toxiproxy
+
+BUNDLED WITH
+   2.2.22

data/gemfiles/activerecord_6.1.gemfile

@@ -0,0 +1,7 @@
+# This file was generated by Appraisal
+
+source "https://rubygems.org"
+
+gem "activerecord", "6.1.0"
+
+gemspec path: "../"

data/gemfiles/activerecord_6.1.gemfile.lock

@@ -0,0 +1,66 @@
+PATH
+  remote: ..
+  specs:
+    lhm-shopify (3.5.1)
+      retriable (>= 3.0.0)
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    activemodel (6.1.0)
+      activesupport (= 6.1.0)
+    activerecord (6.1.0)
+      activemodel (= 6.1.0)
+      activesupport (= 6.1.0)
+    activesupport (6.1.0)
+      concurrent-ruby (~> 1.0, >= 1.0.2)
+      i18n (>= 1.6, < 2)
+      minitest (>= 5.1)
+      tzinfo (~> 2.0)
+      zeitwerk (~> 2.3)
+    after_do (0.4.0)
+    appraisal (2.4.1)
+      bundler
+      rake
+      thor (>= 0.14.0)
+    byebug (11.1.3)
+    concurrent-ruby (1.1.9)
+    docile (1.4.0)
+    i18n (1.8.11)
+      concurrent-ruby (~> 1.0)
+    minitest (5.14.4)
+    mocha (1.13.0)
+    mysql2 (0.5.3)
+    rake (13.0.6)
+    retriable (3.1.2)
+    simplecov (0.21.2)
+      docile (~> 1.1)
+      simplecov-html (~> 0.11)
+      simplecov_json_formatter (~> 0.1)
+    simplecov-html (0.12.3)
+    simplecov_json_formatter (0.1.3)
+    thor (1.1.0)
+    toxiproxy (2.0.0)
+    tzinfo (2.0.4)
+      concurrent-ruby (~> 1.0)
+    zeitwerk (2.5.1)
+
+PLATFORMS
+  x86_64-darwin-20
+  x86_64-linux
+
+DEPENDENCIES
+  activerecord (= 6.1.0)
+  after_do
+  appraisal
+  byebug
+  lhm-shopify!
+  minitest
+  mocha
+  mysql2
+  rake
+  simplecov
+  toxiproxy
+
+BUNDLED WITH
+   2.2.22

data/gemfiles/activerecord_7.0.0.alpha2.gemfile

@@ -0,0 +1,7 @@
+# This file was generated by Appraisal
+
+source "https://rubygems.org"
+
+gem "activerecord", "7.0.0.alpha2"
+
+gemspec path: "../"

data/gemfiles/activerecord_7.0.0.alpha2.gemfile.lock

@@ -0,0 +1,64 @@
+PATH
+  remote: ..
+  specs:
+    lhm-shopify (3.5.1)
+      retriable (>= 3.0.0)
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    activemodel (7.0.0.alpha2)
+      activesupport (= 7.0.0.alpha2)
+    activerecord (7.0.0.alpha2)
+      activemodel (= 7.0.0.alpha2)
+      activesupport (= 7.0.0.alpha2)
+    activesupport (7.0.0.alpha2)
+      concurrent-ruby (~> 1.0, >= 1.0.2)
+      i18n (>= 1.6, < 2)
+      minitest (>= 5.1)
+      tzinfo (~> 2.0)
+    after_do (0.4.0)
+    appraisal (2.4.1)
+      bundler
+      rake
+      thor (>= 0.14.0)
+    byebug (11.1.3)
+    concurrent-ruby (1.1.9)
+    docile (1.4.0)
+    i18n (1.8.11)
+      concurrent-ruby (~> 1.0)
+    minitest (5.14.4)
+    mocha (1.13.0)
+    mysql2 (0.5.3)
+    rake (13.0.6)
+    retriable (3.1.2)
+    simplecov (0.21.2)
+      docile (~> 1.1)
+      simplecov-html (~> 0.11)
+      simplecov_json_formatter (~> 0.1)
+    simplecov-html (0.12.3)
+    simplecov_json_formatter (0.1.3)
+    thor (1.1.0)
+    toxiproxy (2.0.0)
+    tzinfo (2.0.4)
+      concurrent-ruby (~> 1.0)
+
+PLATFORMS
+  x86_64-darwin-20
+  x86_64-linux
+
+DEPENDENCIES
+  activerecord (= 7.0.0.alpha2)
+  after_do
+  appraisal
+  byebug
+  lhm-shopify!
+  minitest
+  mocha
+  mysql2
+  rake
+  simplecov
+  toxiproxy
+
+BUNDLED WITH
+   2.2.22

data/lhm.gemspec

@@ -1,6 +1,6 @@
-# -*- encoding: utf-8 -*-
+# frozen_string_literal: true
 
-lib = File.expand_path('../lib', __FILE__)
+lib = File.expand_path('lib', __dir__)
 $:.unshift(lib) unless $:.include?(lib)
 
 require 'lhm/version'
@@ -25,10 +25,14 @@ Gem::Specification.new do |s|
 
   s.add_dependency 'retriable', '>= 3.0.0'
 
+  s.add_development_dependency 'activerecord'
   s.add_development_dependency 'minitest'
   s.add_development_dependency 'mocha'
+  s.add_development_dependency 'after_do'
   s.add_development_dependency 'rake'
-  s.add_development_dependency 'activerecord'
   s.add_development_dependency 'mysql2'
   s.add_development_dependency 'simplecov'
+  s.add_development_dependency 'toxiproxy'
+  s.add_development_dependency 'appraisal'
+  s.add_development_dependency 'byebug'
 end

data/lib/lhm/atomic_switcher.rb

@@ -39,7 +39,7 @@ module Lhm
     private
 
     def execute
-      @connection.execute(atomic_switch, @retry_options)
+      @connection.execute(atomic_switch, should_retry: true, retry_options: @retry_options)
     end
   end
 end

data/lib/lhm/chunk_insert.rb

@@ -1,4 +1,5 @@
 require 'lhm/sql_retry'
+require 'lhm/proxysql_helper'
 
 module Lhm
   class ChunkInsert
@@ -11,7 +12,7 @@ module Lhm
     end
 
     def insert_and_return_count_of_rows_created
-      @connection.update(sql, @retry_options)
+      @connection.update(sql, should_retry: true, retry_options: @retry_options)
     end
 
     def sql

data/lib/lhm/chunker.rb

@@ -31,7 +31,7 @@ module Lhm
       @retry_options = options[:retriable] || {}
       @retry_helper = SqlRetry.new(
         @connection,
-        {
+        options: {
           log_prefix: "Chunker"
         }.merge!(@retry_options)
       )
@@ -79,7 +79,7 @@ module Lhm
     private
 
     def raise_on_non_pk_duplicate_warning
-      @connection.execute("show warnings", @retry_options).each do |level, code, message|
+      @connection.execute("show warnings", should_retry: true, retry_options: @retry_options).each do |level, code, message|
         unless message.match?(/Duplicate entry .+ for key 'PRIMARY'/)
           m = "Unexpected warning found for inserted row: #{message}"
           Lhm.logger.warn(m)
@@ -101,7 +101,7 @@ module Lhm
 
     def upper_id(next_id, stride)
       sql = "select id from `#{ @migration.origin_name }` where id >= #{ next_id } order by id limit 1 offset #{ stride - 1}"
-      top = @connection.select_value(sql, @retry_options)
+      top = @connection.select_value(sql, should_retry: true, retry_options: @retry_options)
 
       [top ? top.to_i : @limit, @limit].min
     end