lhm-shopify 3.5.0 → 3.5.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 6028b7c6d8d1f2e2d8b786ae3e8f41c4525c7507529085c3427a8dc9cf11bd2e
-  data.tar.gz: de625eda2736c56cfa813ba73f8447f6e798c30080274d91164b895879a7f3a7
+  metadata.gz: 4f676427b10052113b08c4dda8f648bc11234024deab2f1e96de332e0906a110
+  data.tar.gz: dbd91d1b72d713e9d6e18edb46900971ee89c20563d5e93db9eec746887a12d7
 SHA512:
-  metadata.gz: 6561aadaa26f6f28f4fd04987a7fca180e9130194cd756cc5a53a654e060b7bd93efaec78a6b067d39e2b31fd6b4f1826a4de9ac90da162af912b1dd101b8732
-  data.tar.gz: b221afa7215cf36f66fdd48ffd449a2d6034c8710c36d9ccdbf909b2078b83fc610fa1c1170214752371e50f41ed6504186975c80e5a8164ef6b25dce631fe89
+  metadata.gz: be5dc9f5d6d2a3ae62b476fa19455bd0f864b989d6ffbff35a93a0afea4e8d9be4492a4b3bfc2bec1029193114187eaf4eda5bb6dfde9d0be34884bde317c1c1
+  data.tar.gz: aa4e866f74da3255586c1597ae63a3792606bd092e46c3a90335d22b9f6c9b243bc133d99e50233c884b03064dfaeee75983597f84733484e05ec456833f3395

data/.github/workflows/test.yml

@@ -10,15 +10,28 @@ jobs:
   test:
     name: Run tests
     runs-on: ubuntu-latest
+
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - ruby-version: "2.4"
+            activerecord-version: "5.2"
+          - ruby-version: "2.5"
+            activerecord-version: "6.0"
+          - ruby-version: "2.7.5"
+            activerecord-version: "6.1"
+          - ruby-version: "3.0"
+            activerecord-version: "7.0.0.alpha2"
     steps:
       - uses: actions/checkout@v2
       - name: Set up Ruby
         uses: ruby/setup-ruby@v1
         with:
-          ruby-version: 2.5.1
-          bundler-cache: true
+          ruby-version: ${{matrix.ruby-version}}
+          bundler-cache: false
       - name: Install Ruby packages
-        run: bundle install
+        run: BUNDLE_GEMFILE="${GITHUB_WORKSPACE}/gemfiles/activerecord_${{ matrix.activerecord-version }}.gemfile" bundle install
       - name: Install Ubuntu packages
         run: sudo apt-get update && sudo apt-get install numactl libaio-dev libmysqlclient-dev
       - name: Setup MySQL and ProxySQL (docker-compose)
@@ -27,4 +40,4 @@ jobs:
         run: ./scripts/helpers/wait-for-dbs.sh
         timeout-minutes: 2
       - name: Run tests
-        run: bundle exec rake specs
+        run: BUNDLE_GEMFILE="${GITHUB_WORKSPACE}/gemfiles/activerecord_${{ matrix.activerecord-version }}.gemfile" bundle exec rake specs

data/.gitignore

@@ -1,6 +1,5 @@
 *.gem
 .bundle
-gemfiles/*.lock
 pkg/*
 .rvmrc
 .ruby-version
@@ -11,4 +10,3 @@ omg.ponies
 *~
 coverage
 .idea/
-Gemfile.lock

data/Appraisals

@@ -0,0 +1,24 @@
+# Oldest supported Rails version
+appraise "activerecord-5.2" do
+  gem "activerecord", "5.2.6"
+
+# This gemfile will be ran against ruby 2.4 and some gems might require 2.4+. The following list are the gems
+# and their respective versions that work with ruby <= 2.4
+  gem "simplecov", "0.18.5"
+  gem "docile", "1.3.5"
+end
+
+# First conflicted version
+appraise "activerecord-6.0" do
+  gem "activerecord", "6.0.0"
+end
+
+# Second conflicted version
+appraise "activerecord-6.1" do
+  gem "activerecord", "6.1.0"
+end
+
+# Latest version at the moment
+appraise "activerecord-7.0.0.alpha2" do
+  gem "activerecord", "7.0.0.alpha2"
+end

data/CHANGELOG.md

@@ -1,3 +1,26 @@
+# 3.5.4 (Dec, 2021)
+* Refactored the way options are handled internally. Code is now much clearer to understand
+* Removed optional connection_options from `Lhm.setup` and `Lhm.connection`
+* Option `reconnect_with_consistent_host` will now be provided with `options` for `Lhm.change_table`
+
+# 3.5.3 (Dec, 2021)
+* Adds ProxySQL comments at the end of query to accommodate for internal tool's requirements
+
+# 3.5.2 (Dec, 2021)
+* Fixed error on undefined connection, when calling `Lhm.connection` without calling `Lhm.setup` first
+* Changed `Lhm.connection.connection` to `lhm.connection.ar_connection` for increased clarity and readability
+
+# 3.5.1 (Dec , 2021)
+* Add better logging to the LHM components (https://github.com/Shopify/lhm/pull/112)
+* Slave lag throttler now supports ActiveRecord > 6.0
+* [Dev] Add `Appraisals` to test against multiple version
+
+# 3.5.0 (Dec , 2021)
+* Duplicate of 3.4.2 (unfortunate mistake)
+
+# 3.4.2 (Sept, 2021)
+* Fixed Chunker's undefined name error (https://github.com/Shopify/lhm/pull/110)
+
 # 3.4.1 (Sep 22, 2021)
 
 * Add better logging to the LHM components (https://github.com/Shopify/lhm/pull/108)

data/Gemfile.lock

@@ -0,0 +1,66 @@
+PATH
+  remote: .
+  specs:
+    lhm-shopify (3.5.4)
+      retriable (>= 3.0.0)
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    activemodel (6.1.4.1)
+      activesupport (= 6.1.4.1)
+    activerecord (6.1.4.1)
+      activemodel (= 6.1.4.1)
+      activesupport (= 6.1.4.1)
+    activesupport (6.1.4.1)
+      concurrent-ruby (~> 1.0, >= 1.0.2)
+      i18n (>= 1.6, < 2)
+      minitest (>= 5.1)
+      tzinfo (~> 2.0)
+      zeitwerk (~> 2.3)
+    after_do (0.4.0)
+    appraisal (2.4.1)
+      bundler
+      rake
+      thor (>= 0.14.0)
+    byebug (11.1.3)
+    concurrent-ruby (1.1.9)
+    docile (1.4.0)
+    i18n (1.8.11)
+      concurrent-ruby (~> 1.0)
+    minitest (5.14.4)
+    mocha (1.13.0)
+    mysql2 (0.5.3)
+    rake (13.0.6)
+    retriable (3.1.2)
+    simplecov (0.21.2)
+      docile (~> 1.1)
+      simplecov-html (~> 0.11)
+      simplecov_json_formatter (~> 0.1)
+    simplecov-html (0.12.3)
+    simplecov_json_formatter (0.1.3)
+    thor (1.1.0)
+    toxiproxy (2.0.0)
+    tzinfo (2.0.4)
+      concurrent-ruby (~> 1.0)
+    zeitwerk (2.5.1)
+
+PLATFORMS
+  x86_64-darwin-20
+  x86_64-linux
+
+DEPENDENCIES
+  activerecord
+  after_do
+  appraisal
+  byebug
+  lhm-shopify!
+  minitest
+  mocha
+  mysql2
+  rake
+  simplecov
+  toxiproxy
+
+BUNDLED WITH
+   2.2.22

data/README.md

@@ -109,6 +109,50 @@ end
 to prevent accidental data loss. After successful or failed LHM migrations, these leftover
 tables must be cleaned up.
 
+### Usage with ProxySQL
+LHM can recover from connection loss. However, when used in conjunction with ProxySQL, there are multiple ways that
+connection loss could induce data loss (if triggered by a failover). Therefore  it will perform additional checks to
+ensure that the MySQL host stays consistent across the schema migrations if the feature is enabled. 
+This is done by tagging every query with `/*maintenance:lhm*/`, which will be recognized by ProxySQL.
+However, to get this feature working, a new ProxySQL query rule must be added.
+```cnf
+{
+  rule_id = <rule id>
+  active = 1
+  match_pattern = "maintenance:lhm"
+  destination_hostgroup = <MySQL writer's hostgroup>
+}
+```
+
+This will ensure that all relevant queries are forwarded to the current writer.
+
+Also, ProxySQL disables [multiplexing](https://proxysql.com/documentation/multiplexing/) for `select` on `@@` variables.
+Therefore, the following rules must be added to ensure that queries (even if tagged with `/*maintenance:lhm*/`) get 
+forwarded to the right target.
+```cnf
+{
+  rule_id = <rule id>
+  active = 1
+  match_digest = "@@global\.server_id"
+  multiplex = 2
+},
+{
+  rule_id = <rule id>
+  active = 1
+  match_digest = "@@global\.hostname"
+  multiplex = 2
+}
+```
+
+Once these changes are added to the ProxySQL configuration (either through `.cnf` or dynamically through the admin interface), 
+the feature can be enabled. This is done by adding this flag when providing options to the migration:
+```ruby
+ Lhm.change_table(..., options: {reconnect_with_consistent_host: true}) do |t|
+  ...
+end
+```
+**Note**: This feature is disabled by default
+
 ## Throttler
 
 LHM uses a throttling mechanism to read data in your original table. By default, 2,000 rows are read each 0.1 second. If you want to change that behaviour, you can pass an instance of a throttler with the `throttler` option. In this example, 1,000 rows will be read with a 10 second delay between each processing:
@@ -259,6 +303,15 @@ COV=1 bundle exec rake unit && bundle exec rake integration
 open coverage/index.html
 ```
 
+### Merging for a new version
+When creating a PR for a new version, make sure that th version has been bumped in `lib/lhm/version.rb`. Then run the following code snippet to ensure the everything is consistent, otherwise 
+the gem will not publish.
+```bash
+bundle install
+bundle update
+bundle exec appraisals install
+```
+
 ### Docker Compose
 The integration tests rely on a replication configuration for MySQL which is being proxied by an instance of ProxySQL.
 It is important that every container is running to execute the integration test suite.

data/Rakefile

@@ -16,15 +16,16 @@ Rake::TestTask.new('integration') do |t|
   t.libs << 'spec'
   t.test_files = FileList['spec/integration/**/*_spec.rb']
   t.verbose = true
-  end
+end
 
 Rake::TestTask.new('dev') do |t|
   t.libs << 'lib'
   t.libs << 'spec'
-  t.test_files = FileList[
-    'spec/test_helper.rb',
-  #  Add file to test individually
-  ]
+
+  files = FileList.new('spec/test_helper.rb')
+  files.add(ENV["SINGLE_TEST"]) if ENV["SINGLE_TEST"]
+  t.test_files = files
+
   t.verbose = true
 end
 

data/dev.yml

@@ -5,12 +5,16 @@ up:
           or: [mysql@5.7]
           conflicts: [shopify/shopify/mysql-client, mysql-connector-c, mysql, mysql-client]
       - wget
-  - ruby: 2.6.5
+  - ruby: 2.7.5
   - bundler
+  - custom:
+      name: Get Appraisal gems
+      met?: bundle exec appraisal install
+      meet: ":"
   - docker
   - custom:
       name: Docker Compose
-      met?: docker compose ls | grep -ioE -q "lhm.*running\(3\)"
+      met?: docker compose ls | grep -ioE -q "lhm.*running\(4\)"
       meet: docker compose up -d
   - custom:
       name: Waiting for DBs to be operational
@@ -20,8 +24,19 @@ up:
 commands:
   unit: bundle exec rake unit
   int: bundle exec rake integration
-  test: bundle exec rake unit && bundle exec rake integration
+  test:
+    optional: file
+    aliases: [ t ]
+    run: |
+      if [[ $# -eq 0 ]]; then
+        bundle exec rake unit && bundle exec rake integration
+      else
+        SINGLE_TEST="$@" bundle exec rake dev
+      fi
+  appraisals: bundle exec appraisal rake specs
   cov: rm -rf coverage; COV=1 bundle exec rake unit && bundle exec rake integration; open coverage/index.html
   logs:
     desc: "See the DB logs (ctrl-c + ctrl-c to exit)"
     run: docker-compose logs -f
+  clear:
+    run: docker-compose rm -v -s -f && docker-compose up -d && ./scripts/helpers/wait-for-dbs.sh

data/docker-compose.yml

@@ -1,14 +1,16 @@
 services:
-#  Writer
+  #  Writer
   mysql-1:
     container_name: mysql-1
     image: percona:5.7
     command:
       --server-id=1
+      --log-bin
       --log-slave-updates=ON
       --gtid-mode=ON
       --enforce-gtid-consistency=ON
       --read-only=OFF
+      --max-connections=1000
     hostname: 'mysql-1'
     volumes:
       - ./scripts/mysql/writer:/docker-entrypoint-initdb.d
@@ -17,16 +19,18 @@ services:
       MYSQL_HOST: mysql-1
     ports:
       - "33006:3306"
-#  Reader
+  #  Reader
   mysql-2:
     container_name: mysql-2
     image: percona:5.7
     command:
       --server-id=2
+      --log-bin
       --log-slave-updates=ON
       --gtid-mode=ON
       --enforce-gtid-consistency=ON
       --read-only=ON
+      --max-connections=1000
     hostname: 'mysql-2'
     volumes:
       - ./scripts/mysql/reader:/docker-entrypoint-initdb.d
@@ -35,7 +39,7 @@ services:
       MYSQL_HOST: mysql-2
     ports:
       - "33007:3306"
-#  Proxysql
+  #  Proxysql
   proxysql:
     container_name: proxysql
     image: proxysql/proxysql:2.0.11
@@ -44,3 +48,11 @@ services:
     command: "proxysql -c /etc/proxysql.cnf -f --idle-threads"
     ports:
       - "33005:3306"
+      - "6032:6032"
+  toxiproxy:
+    container_name: toxiproxy
+    image: "ghcr.io/shopify/toxiproxy"
+    ports:
+      - "8474:8474"
+      - "22220:22220"
+      - "22222:22222"

data/gemfiles/activerecord_5.2.gemfile

@@ -0,0 +1,9 @@
+# This file was generated by Appraisal
+
+source "https://rubygems.org"
+
+gem "activerecord", "5.2.6"
+gem "simplecov", "0.18.5"
+gem "docile", "1.3.5"
+
+gemspec path: "../"

data/gemfiles/activerecord_5.2.gemfile.lock

@@ -0,0 +1,65 @@
+PATH
+  remote: ..
+  specs:
+    lhm-shopify (3.5.4)
+      retriable (>= 3.0.0)
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    activemodel (5.2.6)
+      activesupport (= 5.2.6)
+    activerecord (5.2.6)
+      activemodel (= 5.2.6)
+      activesupport (= 5.2.6)
+      arel (>= 9.0)
+    activesupport (5.2.6)
+      concurrent-ruby (~> 1.0, >= 1.0.2)
+      i18n (>= 0.7, < 2)
+      minitest (~> 5.1)
+      tzinfo (~> 1.1)
+    after_do (0.4.0)
+    appraisal (2.4.1)
+      bundler
+      rake
+      thor (>= 0.14.0)
+    arel (9.0.0)
+    byebug (11.1.3)
+    concurrent-ruby (1.1.9)
+    docile (1.3.5)
+    i18n (1.8.11)
+      concurrent-ruby (~> 1.0)
+    minitest (5.14.4)
+    mocha (1.13.0)
+    mysql2 (0.5.3)
+    rake (13.0.6)
+    retriable (3.1.2)
+    simplecov (0.18.5)
+      docile (~> 1.1)
+      simplecov-html (~> 0.11)
+    simplecov-html (0.12.3)
+    thor (1.1.0)
+    thread_safe (0.3.6)
+    toxiproxy (2.0.0)
+    tzinfo (1.2.9)
+      thread_safe (~> 0.1)
+
+PLATFORMS
+  x86_64-darwin-20
+
+DEPENDENCIES
+  activerecord (= 5.2.6)
+  after_do
+  appraisal
+  byebug
+  docile (= 1.3.5)
+  lhm-shopify!
+  minitest
+  mocha
+  mysql2
+  rake
+  simplecov (= 0.18.5)
+  toxiproxy
+
+BUNDLED WITH
+   2.2.22

data/gemfiles/activerecord_6.0.gemfile

@@ -0,0 +1,7 @@
+# This file was generated by Appraisal
+
+source "https://rubygems.org"
+
+gem "activerecord", "6.0.0"
+
+gemspec path: "../"

data/gemfiles/activerecord_6.0.gemfile.lock

@@ -0,0 +1,67 @@
+PATH
+  remote: ..
+  specs:
+    lhm-shopify (3.5.4)
+      retriable (>= 3.0.0)
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    activemodel (6.0.0)
+      activesupport (= 6.0.0)
+    activerecord (6.0.0)
+      activemodel (= 6.0.0)
+      activesupport (= 6.0.0)
+    activesupport (6.0.0)
+      concurrent-ruby (~> 1.0, >= 1.0.2)
+      i18n (>= 0.7, < 2)
+      minitest (~> 5.1)
+      tzinfo (~> 1.1)
+      zeitwerk (~> 2.1, >= 2.1.8)
+    after_do (0.4.0)
+    appraisal (2.4.1)
+      bundler
+      rake
+      thor (>= 0.14.0)
+    byebug (11.1.3)
+    concurrent-ruby (1.1.9)
+    docile (1.4.0)
+    i18n (1.8.11)
+      concurrent-ruby (~> 1.0)
+    minitest (5.14.4)
+    mocha (1.13.0)
+    mysql2 (0.5.3)
+    rake (13.0.6)
+    retriable (3.1.2)
+    simplecov (0.21.2)
+      docile (~> 1.1)
+      simplecov-html (~> 0.11)
+      simplecov_json_formatter (~> 0.1)
+    simplecov-html (0.12.3)
+    simplecov_json_formatter (0.1.3)
+    thor (1.1.0)
+    thread_safe (0.3.6)
+    toxiproxy (2.0.0)
+    tzinfo (1.2.9)
+      thread_safe (~> 0.1)
+    zeitwerk (2.5.1)
+
+PLATFORMS
+  x86_64-darwin-20
+  x86_64-linux
+
+DEPENDENCIES
+  activerecord (= 6.0.0)
+  after_do
+  appraisal
+  byebug
+  lhm-shopify!
+  minitest
+  mocha
+  mysql2
+  rake
+  simplecov
+  toxiproxy
+
+BUNDLED WITH
+   2.2.22

data/gemfiles/activerecord_6.1.gemfile

@@ -0,0 +1,7 @@
+# This file was generated by Appraisal
+
+source "https://rubygems.org"
+
+gem "activerecord", "6.1.0"
+
+gemspec path: "../"

data/gemfiles/activerecord_6.1.gemfile.lock

@@ -0,0 +1,66 @@
+PATH
+  remote: ..
+  specs:
+    lhm-shopify (3.5.4)
+      retriable (>= 3.0.0)
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    activemodel (6.1.0)
+      activesupport (= 6.1.0)
+    activerecord (6.1.0)
+      activemodel (= 6.1.0)
+      activesupport (= 6.1.0)
+    activesupport (6.1.0)
+      concurrent-ruby (~> 1.0, >= 1.0.2)
+      i18n (>= 1.6, < 2)
+      minitest (>= 5.1)
+      tzinfo (~> 2.0)
+      zeitwerk (~> 2.3)
+    after_do (0.4.0)
+    appraisal (2.4.1)
+      bundler
+      rake
+      thor (>= 0.14.0)
+    byebug (11.1.3)
+    concurrent-ruby (1.1.9)
+    docile (1.4.0)
+    i18n (1.8.11)
+      concurrent-ruby (~> 1.0)
+    minitest (5.14.4)
+    mocha (1.13.0)
+    mysql2 (0.5.3)
+    rake (13.0.6)
+    retriable (3.1.2)
+    simplecov (0.21.2)
+      docile (~> 1.1)
+      simplecov-html (~> 0.11)
+      simplecov_json_formatter (~> 0.1)
+    simplecov-html (0.12.3)
+    simplecov_json_formatter (0.1.3)
+    thor (1.1.0)
+    toxiproxy (2.0.0)
+    tzinfo (2.0.4)
+      concurrent-ruby (~> 1.0)
+    zeitwerk (2.5.1)
+
+PLATFORMS
+  x86_64-darwin-20
+  x86_64-linux
+
+DEPENDENCIES
+  activerecord (= 6.1.0)
+  after_do
+  appraisal
+  byebug
+  lhm-shopify!
+  minitest
+  mocha
+  mysql2
+  rake
+  simplecov
+  toxiproxy
+
+BUNDLED WITH
+   2.2.22

data/gemfiles/activerecord_7.0.0.alpha2.gemfile

@@ -0,0 +1,7 @@
+# This file was generated by Appraisal
+
+source "https://rubygems.org"
+
+gem "activerecord", "7.0.0.alpha2"
+
+gemspec path: "../"

data/gemfiles/activerecord_7.0.0.alpha2.gemfile.lock

@@ -0,0 +1,64 @@
+PATH
+  remote: ..
+  specs:
+    lhm-shopify (3.5.4)
+      retriable (>= 3.0.0)
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    activemodel (7.0.0.alpha2)
+      activesupport (= 7.0.0.alpha2)
+    activerecord (7.0.0.alpha2)
+      activemodel (= 7.0.0.alpha2)
+      activesupport (= 7.0.0.alpha2)
+    activesupport (7.0.0.alpha2)
+      concurrent-ruby (~> 1.0, >= 1.0.2)
+      i18n (>= 1.6, < 2)
+      minitest (>= 5.1)
+      tzinfo (~> 2.0)
+    after_do (0.4.0)
+    appraisal (2.4.1)
+      bundler
+      rake
+      thor (>= 0.14.0)
+    byebug (11.1.3)
+    concurrent-ruby (1.1.9)
+    docile (1.4.0)
+    i18n (1.8.11)
+      concurrent-ruby (~> 1.0)
+    minitest (5.14.4)
+    mocha (1.13.0)
+    mysql2 (0.5.3)
+    rake (13.0.6)
+    retriable (3.1.2)
+    simplecov (0.21.2)
+      docile (~> 1.1)
+      simplecov-html (~> 0.11)
+      simplecov_json_formatter (~> 0.1)
+    simplecov-html (0.12.3)
+    simplecov_json_formatter (0.1.3)
+    thor (1.1.0)
+    toxiproxy (2.0.0)
+    tzinfo (2.0.4)
+      concurrent-ruby (~> 1.0)
+
+PLATFORMS
+  x86_64-darwin-20
+  x86_64-linux
+
+DEPENDENCIES
+  activerecord (= 7.0.0.alpha2)
+  after_do
+  appraisal
+  byebug
+  lhm-shopify!
+  minitest
+  mocha
+  mysql2
+  rake
+  simplecov
+  toxiproxy
+
+BUNDLED WITH
+   2.2.22