lhm-shopify 3.4.2 → 3.5.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: cfb1802db81b2673ea5e52acec84b4e457a8d5debb74f5391fcb282bf1fc8484
-  data.tar.gz: 88abd0ae0484efa69777b0284c253788996b2148c0cb61fc035c0ccd7d49627c
+  metadata.gz: ef039ab1af9028a875f5851d7b7bd21dc74f363906037cbbe3acc869b008221e
+  data.tar.gz: 03a9d41b63437471f2edaf2faacd1f4d543440aebc64ebe53e2445986b52eb2c
 SHA512:
-  metadata.gz: 4b98a803678e1ab543997d4e6f9031b02511324293c2d97b3f94938ae6b701e55c122633ab6044920363c56d9746e35f9b131433eb65108437c4ffeb8ca2e5ea
-  data.tar.gz: '018c3a179dfdf90f9b5fd4bd3f67636644fc47eda7b0b825aee00330db0f83a59cf6855a5e18105493a89ac0a372ac26fd2c356148282449ca2de84fc56c5179'
+  metadata.gz: 768f2438270cd0f1cfe2d3e624068cb8c45bf083615cdd936e4598db8a5c831c112983f41918ba0e82ce4ee9ecdd3728e53926251512a18b43c94f04afb14a8a
+  data.tar.gz: 94e677823ae472e1db287f250481cf63e8ac0a2f8d85018efb2b07b6ca197d2e4c0f8f5ea7a539d222fbd7928187e61957cd26a95dfc0ad2f39f5e8e27d45a88

data/.github/workflows/test.yml

@@ -10,25 +10,34 @@ jobs:
   test:
     name: Run tests
     runs-on: ubuntu-latest
+
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - ruby-version: "2.4"
+            activerecord-version: "5.2"
+          - ruby-version: "2.5"
+            activerecord-version: "6.0"
+          - ruby-version: "2.7.5"
+            activerecord-version: "6.1"
+          - ruby-version: "3.0"
+            activerecord-version: "7.0.0.alpha2"
     steps:
       - uses: actions/checkout@v2
       - name: Set up Ruby
         uses: ruby/setup-ruby@v1
         with:
-          ruby-version: 2.5.1
-          bundler-cache: true
-      - name: Cache MySQL
-        uses: actions/cache@v2
-        with:
-          path: dbdeployer/binaries
-          key: ${{ runner.os }}-dbdeployer-${{ hashFiles('**/dbdeployer/install.sh') }}
+          ruby-version: ${{matrix.ruby-version}}
+          bundler-cache: false
       - name: Install Ruby packages
-        run: bundle install
+        run: BUNDLE_GEMFILE="${GITHUB_WORKSPACE}/gemfiles/activerecord_${{ matrix.activerecord-version }}.gemfile" bundle install
       - name: Install Ubuntu packages
-        run: sudo apt-get install numactl libaio-dev
-      - name: Setup MySQL
-        run: ./dbdeployer/install.sh
-      - name: Run integration tests
-        run: bundle exec rake integration
-      - name: Run unit tests
-        run: bundle exec rake unit
+        run: sudo apt-get update && sudo apt-get install numactl libaio-dev libmysqlclient-dev
+      - name: Setup MySQL and ProxySQL (docker-compose)
+        run: docker-compose up -d # Might have to change to docker compose up -d (i.e. Compose V2) when the Ubuntu image changes the docker-compose version
+      - name: Wait until DBs are alive
+        run: ./scripts/helpers/wait-for-dbs.sh
+        timeout-minutes: 2
+      - name: Run tests
+        run: BUNDLE_GEMFILE="${GITHUB_WORKSPACE}/gemfiles/activerecord_${{ matrix.activerecord-version }}.gemfile" bundle exec rake specs

data/.gitignore

@@ -1,17 +1,12 @@
 *.gem
 .bundle
-gemfiles/*.lock
 pkg/*
 .rvmrc
 .ruby-version
 .ruby-gemset
 bin/rake
-spec/integration/database.yml
 gemfiles/vendor
 omg.ponies
 *~
 coverage
-bin/dbdeployer
-/dbdeployer/sandboxes
-/dbdeployer/binaries
-Gemfile.lock
+.idea/

data/Appraisals

@@ -0,0 +1,24 @@
+# Oldest supported Rails version
+appraise "activerecord-5.2" do
+  gem "activerecord", "5.2.6"
+
+# This gemfile will be ran against ruby 2.4 and some gems might require 2.4+. The following list are the gems
+# and their respective versions that work with ruby <= 2.4
+  gem "simplecov", "0.18.5"
+  gem "docile", "1.3.5"
+end
+
+# First conflicted version
+appraise "activerecord-6.0" do
+  gem "activerecord", "6.0.0"
+end
+
+# Second conflicted version
+appraise "activerecord-6.1" do
+  gem "activerecord", "6.1.0"
+end
+
+# Latest version at the moment
+appraise "activerecord-7.0.0.alpha2" do
+  gem "activerecord", "7.0.0.alpha2"
+end

data/CHANGELOG.md

@@ -1,3 +1,18 @@
+# 3.5.2 (Dec, 2021)
+* Fixed error on undefined connection, when calling `Lhm.connection` without calling `Lhm.setup` first
+* Changed `Lhm.connection.connection` to `lhm.connection.ar_connection` for increased clarity and readability
+
+# 3.5.1 (Dec , 2021)
+* Add better logging to the LHM components (https://github.com/Shopify/lhm/pull/112)
+* Slave lag throttler now supports ActiveRecord > 6.0
+* [Dev] Add `Appraisals` to test against multiple version
+
+# 3.5.0 (Dec , 2021)
+* Duplicate of 3.4.2 (unfortunate mistake)
+
+# 3.4.2 (Sept, 2021)
+* Fixed Chunker's undefined name error (https://github.com/Shopify/lhm/pull/110)
+
 # 3.4.1 (Sep 22, 2021)
 
 * Add better logging to the LHM components (https://github.com/Shopify/lhm/pull/108)

data/Gemfile.lock

@@ -0,0 +1,66 @@
+PATH
+  remote: .
+  specs:
+    lhm-shopify (3.5.3)
+      retriable (>= 3.0.0)
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    activemodel (6.1.4.1)
+      activesupport (= 6.1.4.1)
+    activerecord (6.1.4.1)
+      activemodel (= 6.1.4.1)
+      activesupport (= 6.1.4.1)
+    activesupport (6.1.4.1)
+      concurrent-ruby (~> 1.0, >= 1.0.2)
+      i18n (>= 1.6, < 2)
+      minitest (>= 5.1)
+      tzinfo (~> 2.0)
+      zeitwerk (~> 2.3)
+    after_do (0.4.0)
+    appraisal (2.4.1)
+      bundler
+      rake
+      thor (>= 0.14.0)
+    byebug (11.1.3)
+    concurrent-ruby (1.1.9)
+    docile (1.4.0)
+    i18n (1.8.11)
+      concurrent-ruby (~> 1.0)
+    minitest (5.14.4)
+    mocha (1.13.0)
+    mysql2 (0.5.3)
+    rake (13.0.6)
+    retriable (3.1.2)
+    simplecov (0.21.2)
+      docile (~> 1.1)
+      simplecov-html (~> 0.11)
+      simplecov_json_formatter (~> 0.1)
+    simplecov-html (0.12.3)
+    simplecov_json_formatter (0.1.3)
+    thor (1.1.0)
+    toxiproxy (2.0.0)
+    tzinfo (2.0.4)
+      concurrent-ruby (~> 1.0)
+    zeitwerk (2.5.1)
+
+PLATFORMS
+  x86_64-darwin-20
+  x86_64-linux
+
+DEPENDENCIES
+  activerecord
+  after_do
+  appraisal
+  byebug
+  lhm-shopify!
+  minitest
+  mocha
+  mysql2
+  rake
+  simplecov
+  toxiproxy
+
+BUNDLED WITH
+   2.2.22

data/README.md

@@ -109,6 +109,48 @@ end
 to prevent accidental data loss. After successful or failed LHM migrations, these leftover
 tables must be cleaned up.
 
+### Usage with ProxySQL
+LHM can recover from connection loss. However, when used in conjunction with ProxySQL, there are multiple ways that
+connection loss could induce data loss (if triggered by a failover). Therefore  it will perform additional checks to
+ensure that the MySQL host stays consistent across the schema migrations if the feature is enabled. 
+This is done by tagging every query with `/*maintenance:lhm*/`, which will be recognized by ProxySQL. 
+However, to get this feature working, a new ProxySQL query rule must be added.
+```cnf
+{
+  rule_id = <rule id>
+  active = 1
+  match_pattern = "maintenance:lhm"
+  destination_hostgroup = <MySQL writer's hostgroup>
+}
+```
+
+This will ensure that all relevant queries are forwarded to the current writer.
+
+Also, ProxySQL disables [multiplexing](https://proxysql.com/documentation/multiplexing/) for `select` on `@@` variables.
+Therefore, the following rules must be added to ensure that queries (even if tagged with `/*maintenance:lhm*/`) get 
+forwarded to the right target.
+```cnf
+{
+  rule_id = <rule id>
+  active = 1
+  match_digest = "@@global\.server_id"
+  multiplex = 2
+},
+{
+  rule_id = <rule id>
+  active = 1
+  match_digest = "@@global\.hostname"
+  multiplex = 2
+}
+```
+
+Once these changes are added to the ProxySQL configuration (either through `.cnf` or dynamically through the admin interface), 
+the feature can be enabled. This is done by adding this flag when doing the initial setup:
+```ruby
+ Lhm.setup(connection, options: {reconnect_with_consistent_host: true})
+```
+**Note**: This feature is disabled by default
+
 ## Throttler
 
 LHM uses a throttling mechanism to read data in your original table. By default, 2,000 rows are read each 0.1 second. If you want to change that behaviour, you can pass an instance of a throttler with the `throttler` option. In this example, 1,000 rows will be read with a 10 second delay between each processing:
@@ -259,11 +301,18 @@ COV=1 bundle exec rake unit && bundle exec rake integration
 open coverage/index.html
 ```
 
-### dbdeployer
+### Merging for a new version
+When creating a PR for a new version, make sure that th version has been bumped in `lib/lhm/version.rb`. Then run the following code snippet to ensure the everything is consistent, otherwise 
+the gem will not publish.
+```bash
+bundle install
+bundle update
+bundle exec appraisals install
+```
 
-The integration tests rely on a replicated configuration for MySQL. We're using 
-[`dbdeployer`](https://github.com/datacharmer/dbdeployer) to manage the test nodes
-required to set up these configurations.
+### Docker Compose
+The integration tests rely on a replication configuration for MySQL which is being proxied by an instance of ProxySQL.
+It is important that every container is running to execute the integration test suite.
 
 ## License
 

data/Rakefile

@@ -16,6 +16,16 @@ Rake::TestTask.new('integration') do |t|
   t.libs << 'spec'
   t.test_files = FileList['spec/integration/**/*_spec.rb']
   t.verbose = true
+  end
+
+Rake::TestTask.new('dev') do |t|
+  t.libs << 'lib'
+  t.libs << 'spec'
+  t.test_files = FileList[
+    'spec/test_helper.rb',
+  #  Add file to test individually
+  ]
+  t.verbose = true
 end
 
 task :specs => [:unit, :integration]

data/dev.yml

@@ -5,16 +5,38 @@ up:
           or: [mysql@5.7]
           conflicts: [shopify/shopify/mysql-client, mysql-connector-c, mysql, mysql-client]
       - wget
-  - ruby: 2.6.5
+  - ruby: 2.7.0
   - bundler
   - custom:
-      name: Database
-      met?: test -f spec/integration/database.yml && test "$(./dbdeployer/sandboxes/rsandbox_5_7_22/status_all | grep on\  | wc -l | xargs echo)" = "2"
-      meet: ./dbdeployer/install.sh
-      down: ./dbdeployer/sandboxes/rsandbox_5_7_22/stop_all
+      name: Get Appraisal gems
+      met?: bundle exec appraisal install
+      meet: ":"
+  - docker
+  - custom:
+      name: Docker Compose
+      met?: docker compose ls | grep -ioE -q "lhm.*running\(4\)"
+      meet: docker compose up -d
+  - custom:
+      name: Waiting for DBs to be operational
+      met?: ./scripts/helpers/wait-for-dbs.sh
+      meet: ":"
 
 commands:
   unit: bundle exec rake unit
   int: bundle exec rake integration
-  test: bundle exec rake unit && bundle exec rake integration
+  test:
+    optional: file
+    aliases: [ t ]
+    run: |
+      if [[ $# -eq 0 ]]; then
+        bundle exec rake unit && bundle exec rake integration
+      else
+        bundle exec rake dev TEST="$@"
+      fi
+  appraisals: bundle exec appraisal rake specs
   cov: rm -rf coverage; COV=1 bundle exec rake unit && bundle exec rake integration; open coverage/index.html
+  logs:
+    desc: "See the DB logs (ctrl-c + ctrl-c to exit)"
+    run: docker-compose logs -f
+  clear:
+    run: docker-compose rm -v -s -f && docker-compose up -d && ./scripts/helpers/wait-for-dbs.sh

data/docker-compose.yml

@@ -0,0 +1,58 @@
+services:
+  #  Writer
+  mysql-1:
+    container_name: mysql-1
+    image: percona:5.7
+    command:
+      --server-id=1
+      --log-bin
+      --log-slave-updates=ON
+      --gtid-mode=ON
+      --enforce-gtid-consistency=ON
+      --read-only=OFF
+      --max-connections=1000
+    hostname: 'mysql-1'
+    volumes:
+      - ./scripts/mysql/writer:/docker-entrypoint-initdb.d
+    environment:
+      MYSQL_ROOT_PASSWORD: password
+      MYSQL_HOST: mysql-1
+    ports:
+      - "33006:3306"
+  #  Reader
+  mysql-2:
+    container_name: mysql-2
+    image: percona:5.7
+    command:
+      --server-id=2
+      --log-bin
+      --log-slave-updates=ON
+      --gtid-mode=ON
+      --enforce-gtid-consistency=ON
+      --read-only=ON
+      --max-connections=1000
+    hostname: 'mysql-2'
+    volumes:
+      - ./scripts/mysql/reader:/docker-entrypoint-initdb.d
+    environment:
+      MYSQL_ROOT_PASSWORD: password
+      MYSQL_HOST: mysql-2
+    ports:
+      - "33007:3306"
+  #  Proxysql
+  proxysql:
+    container_name: proxysql
+    image: proxysql/proxysql:2.0.11
+    volumes:
+      - ./scripts/proxysql/proxysql.cnf:/etc/proxysql.cnf
+    command: "proxysql -c /etc/proxysql.cnf -f --idle-threads"
+    ports:
+      - "33005:3306"
+      - "6032:6032"
+  toxiproxy:
+    container_name: toxiproxy
+    image: "ghcr.io/shopify/toxiproxy"
+    ports:
+      - "8474:8474"
+      - "22220:22220"
+      - "22222:22222"

data/gemfiles/activerecord_5.2.gemfile

@@ -0,0 +1,9 @@
+# This file was generated by Appraisal
+
+source "https://rubygems.org"
+
+gem "activerecord", "5.2.6"
+gem "simplecov", "0.18.5"
+gem "docile", "1.3.5"
+
+gemspec path: "../"

data/gemfiles/activerecord_5.2.gemfile.lock

@@ -0,0 +1,65 @@
+PATH
+  remote: ..
+  specs:
+    lhm-shopify (3.5.3)
+      retriable (>= 3.0.0)
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    activemodel (5.2.6)
+      activesupport (= 5.2.6)
+    activerecord (5.2.6)
+      activemodel (= 5.2.6)
+      activesupport (= 5.2.6)
+      arel (>= 9.0)
+    activesupport (5.2.6)
+      concurrent-ruby (~> 1.0, >= 1.0.2)
+      i18n (>= 0.7, < 2)
+      minitest (~> 5.1)
+      tzinfo (~> 1.1)
+    after_do (0.4.0)
+    appraisal (2.4.1)
+      bundler
+      rake
+      thor (>= 0.14.0)
+    arel (9.0.0)
+    byebug (11.1.3)
+    concurrent-ruby (1.1.9)
+    docile (1.3.5)
+    i18n (1.8.11)
+      concurrent-ruby (~> 1.0)
+    minitest (5.14.4)
+    mocha (1.13.0)
+    mysql2 (0.5.3)
+    rake (13.0.6)
+    retriable (3.1.2)
+    simplecov (0.18.5)
+      docile (~> 1.1)
+      simplecov-html (~> 0.11)
+    simplecov-html (0.12.3)
+    thor (1.1.0)
+    thread_safe (0.3.6)
+    toxiproxy (2.0.0)
+    tzinfo (1.2.9)
+      thread_safe (~> 0.1)
+
+PLATFORMS
+  x86_64-darwin-20
+
+DEPENDENCIES
+  activerecord (= 5.2.6)
+  after_do
+  appraisal
+  byebug
+  docile (= 1.3.5)
+  lhm-shopify!
+  minitest
+  mocha
+  mysql2
+  rake
+  simplecov (= 0.18.5)
+  toxiproxy
+
+BUNDLED WITH
+   2.2.22

data/gemfiles/activerecord_6.0.gemfile

@@ -0,0 +1,7 @@
+# This file was generated by Appraisal
+
+source "https://rubygems.org"
+
+gem "activerecord", "6.0.0"
+
+gemspec path: "../"

data/gemfiles/activerecord_6.0.gemfile.lock

@@ -0,0 +1,67 @@
+PATH
+  remote: ..
+  specs:
+    lhm-shopify (3.5.3)
+      retriable (>= 3.0.0)
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    activemodel (6.0.0)
+      activesupport (= 6.0.0)
+    activerecord (6.0.0)
+      activemodel (= 6.0.0)
+      activesupport (= 6.0.0)
+    activesupport (6.0.0)
+      concurrent-ruby (~> 1.0, >= 1.0.2)
+      i18n (>= 0.7, < 2)
+      minitest (~> 5.1)
+      tzinfo (~> 1.1)
+      zeitwerk (~> 2.1, >= 2.1.8)
+    after_do (0.4.0)
+    appraisal (2.4.1)
+      bundler
+      rake
+      thor (>= 0.14.0)
+    byebug (11.1.3)
+    concurrent-ruby (1.1.9)
+    docile (1.4.0)
+    i18n (1.8.11)
+      concurrent-ruby (~> 1.0)
+    minitest (5.14.4)
+    mocha (1.13.0)
+    mysql2 (0.5.3)
+    rake (13.0.6)
+    retriable (3.1.2)
+    simplecov (0.21.2)
+      docile (~> 1.1)
+      simplecov-html (~> 0.11)
+      simplecov_json_formatter (~> 0.1)
+    simplecov-html (0.12.3)
+    simplecov_json_formatter (0.1.3)
+    thor (1.1.0)
+    thread_safe (0.3.6)
+    toxiproxy (2.0.0)
+    tzinfo (1.2.9)
+      thread_safe (~> 0.1)
+    zeitwerk (2.5.1)
+
+PLATFORMS
+  x86_64-darwin-20
+  x86_64-linux
+
+DEPENDENCIES
+  activerecord (= 6.0.0)
+  after_do
+  appraisal
+  byebug
+  lhm-shopify!
+  minitest
+  mocha
+  mysql2
+  rake
+  simplecov
+  toxiproxy
+
+BUNDLED WITH
+   2.2.22

data/gemfiles/activerecord_6.1.gemfile

@@ -0,0 +1,7 @@
+# This file was generated by Appraisal
+
+source "https://rubygems.org"
+
+gem "activerecord", "6.1.0"
+
+gemspec path: "../"

data/gemfiles/activerecord_6.1.gemfile.lock

@@ -0,0 +1,66 @@
+PATH
+  remote: ..
+  specs:
+    lhm-shopify (3.5.3)
+      retriable (>= 3.0.0)
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    activemodel (6.1.0)
+      activesupport (= 6.1.0)
+    activerecord (6.1.0)
+      activemodel (= 6.1.0)
+      activesupport (= 6.1.0)
+    activesupport (6.1.0)
+      concurrent-ruby (~> 1.0, >= 1.0.2)
+      i18n (>= 1.6, < 2)
+      minitest (>= 5.1)
+      tzinfo (~> 2.0)
+      zeitwerk (~> 2.3)
+    after_do (0.4.0)
+    appraisal (2.4.1)
+      bundler
+      rake
+      thor (>= 0.14.0)
+    byebug (11.1.3)
+    concurrent-ruby (1.1.9)
+    docile (1.4.0)
+    i18n (1.8.11)
+      concurrent-ruby (~> 1.0)
+    minitest (5.14.4)
+    mocha (1.13.0)
+    mysql2 (0.5.3)
+    rake (13.0.6)
+    retriable (3.1.2)
+    simplecov (0.21.2)
+      docile (~> 1.1)
+      simplecov-html (~> 0.11)
+      simplecov_json_formatter (~> 0.1)
+    simplecov-html (0.12.3)
+    simplecov_json_formatter (0.1.3)
+    thor (1.1.0)
+    toxiproxy (2.0.0)
+    tzinfo (2.0.4)
+      concurrent-ruby (~> 1.0)
+    zeitwerk (2.5.1)
+
+PLATFORMS
+  x86_64-darwin-20
+  x86_64-linux
+
+DEPENDENCIES
+  activerecord (= 6.1.0)
+  after_do
+  appraisal
+  byebug
+  lhm-shopify!
+  minitest
+  mocha
+  mysql2
+  rake
+  simplecov
+  toxiproxy
+
+BUNDLED WITH
+   2.2.22

data/gemfiles/activerecord_7.0.0.alpha2.gemfile

@@ -0,0 +1,7 @@
+# This file was generated by Appraisal
+
+source "https://rubygems.org"
+
+gem "activerecord", "7.0.0.alpha2"
+
+gemspec path: "../"

data/gemfiles/activerecord_7.0.0.alpha2.gemfile.lock

@@ -0,0 +1,64 @@
+PATH
+  remote: ..
+  specs:
+    lhm-shopify (3.5.3)
+      retriable (>= 3.0.0)
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    activemodel (7.0.0.alpha2)
+      activesupport (= 7.0.0.alpha2)
+    activerecord (7.0.0.alpha2)
+      activemodel (= 7.0.0.alpha2)
+      activesupport (= 7.0.0.alpha2)
+    activesupport (7.0.0.alpha2)
+      concurrent-ruby (~> 1.0, >= 1.0.2)
+      i18n (>= 1.6, < 2)
+      minitest (>= 5.1)
+      tzinfo (~> 2.0)
+    after_do (0.4.0)
+    appraisal (2.4.1)
+      bundler
+      rake
+      thor (>= 0.14.0)
+    byebug (11.1.3)
+    concurrent-ruby (1.1.9)
+    docile (1.4.0)
+    i18n (1.8.11)
+      concurrent-ruby (~> 1.0)
+    minitest (5.14.4)
+    mocha (1.13.0)
+    mysql2 (0.5.3)
+    rake (13.0.6)
+    retriable (3.1.2)
+    simplecov (0.21.2)
+      docile (~> 1.1)
+      simplecov-html (~> 0.11)
+      simplecov_json_formatter (~> 0.1)
+    simplecov-html (0.12.3)
+    simplecov_json_formatter (0.1.3)
+    thor (1.1.0)
+    toxiproxy (2.0.0)
+    tzinfo (2.0.4)
+      concurrent-ruby (~> 1.0)
+
+PLATFORMS
+  x86_64-darwin-20
+  x86_64-linux
+
+DEPENDENCIES
+  activerecord (= 7.0.0.alpha2)
+  after_do
+  appraisal
+  byebug
+  lhm-shopify!
+  minitest
+  mocha
+  mysql2
+  rake
+  simplecov
+  toxiproxy
+
+BUNDLED WITH
+   2.2.22