lhm-shopify 3.4.2 → 3.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: cfb1802db81b2673ea5e52acec84b4e457a8d5debb74f5391fcb282bf1fc8484
-  data.tar.gz: 88abd0ae0484efa69777b0284c253788996b2148c0cb61fc035c0ccd7d49627c
+  metadata.gz: 6028b7c6d8d1f2e2d8b786ae3e8f41c4525c7507529085c3427a8dc9cf11bd2e
+  data.tar.gz: de625eda2736c56cfa813ba73f8447f6e798c30080274d91164b895879a7f3a7
 SHA512:
-  metadata.gz: 4b98a803678e1ab543997d4e6f9031b02511324293c2d97b3f94938ae6b701e55c122633ab6044920363c56d9746e35f9b131433eb65108437c4ffeb8ca2e5ea
-  data.tar.gz: '018c3a179dfdf90f9b5fd4bd3f67636644fc47eda7b0b825aee00330db0f83a59cf6855a5e18105493a89ac0a372ac26fd2c356148282449ca2de84fc56c5179'
+  metadata.gz: 6561aadaa26f6f28f4fd04987a7fca180e9130194cd756cc5a53a654e060b7bd93efaec78a6b067d39e2b31fd6b4f1826a4de9ac90da162af912b1dd101b8732
+  data.tar.gz: b221afa7215cf36f66fdd48ffd449a2d6034c8710c36d9ccdbf909b2078b83fc610fa1c1170214752371e50f41ed6504186975c80e5a8164ef6b25dce631fe89

data/.github/workflows/test.yml

@@ -17,18 +17,14 @@ jobs:
         with:
           ruby-version: 2.5.1
           bundler-cache: true
-      - name: Cache MySQL
-        uses: actions/cache@v2
-        with:
-          path: dbdeployer/binaries
-          key: ${{ runner.os }}-dbdeployer-${{ hashFiles('**/dbdeployer/install.sh') }}
       - name: Install Ruby packages
         run: bundle install
       - name: Install Ubuntu packages
-        run: sudo apt-get install numactl libaio-dev
-      - name: Setup MySQL
-        run: ./dbdeployer/install.sh
-      - name: Run integration tests
-        run: bundle exec rake integration
-      - name: Run unit tests
-        run: bundle exec rake unit
+        run: sudo apt-get update && sudo apt-get install numactl libaio-dev libmysqlclient-dev
+      - name: Setup MySQL and ProxySQL (docker-compose)
+        run: docker-compose up -d # Might have to change to docker compose up -d (i.e. Compose V2) when the Ubuntu image changes the docker-compose version
+      - name: Wait until DBs are alive
+        run: ./scripts/helpers/wait-for-dbs.sh
+        timeout-minutes: 2
+      - name: Run tests
+        run: bundle exec rake specs

data/.gitignore

@@ -6,12 +6,9 @@ pkg/*
 .ruby-version
 .ruby-gemset
 bin/rake
-spec/integration/database.yml
 gemfiles/vendor
 omg.ponies
 *~
 coverage
-bin/dbdeployer
-/dbdeployer/sandboxes
-/dbdeployer/binaries
+.idea/
 Gemfile.lock

data/README.md

@@ -259,11 +259,9 @@ COV=1 bundle exec rake unit && bundle exec rake integration
 open coverage/index.html
 ```
 
-### dbdeployer
-
-The integration tests rely on a replicated configuration for MySQL. We're using 
-[`dbdeployer`](https://github.com/datacharmer/dbdeployer) to manage the test nodes
-required to set up these configurations.
+### Docker Compose
+The integration tests rely on a replication configuration for MySQL which is being proxied by an instance of ProxySQL.
+It is important that every container is running to execute the integration test suite.
 
 ## License
 

data/Rakefile

@@ -16,6 +16,16 @@ Rake::TestTask.new('integration') do |t|
   t.libs << 'spec'
   t.test_files = FileList['spec/integration/**/*_spec.rb']
   t.verbose = true
+  end
+
+Rake::TestTask.new('dev') do |t|
+  t.libs << 'lib'
+  t.libs << 'spec'
+  t.test_files = FileList[
+    'spec/test_helper.rb',
+  #  Add file to test individually
+  ]
+  t.verbose = true
 end
 
 task :specs => [:unit, :integration]

data/dev.yml

@@ -7,14 +7,21 @@ up:
       - wget
   - ruby: 2.6.5
   - bundler
+  - docker
   - custom:
-      name: Database
-      met?: test -f spec/integration/database.yml && test "$(./dbdeployer/sandboxes/rsandbox_5_7_22/status_all | grep on\  | wc -l | xargs echo)" = "2"
-      meet: ./dbdeployer/install.sh
-      down: ./dbdeployer/sandboxes/rsandbox_5_7_22/stop_all
+      name: Docker Compose
+      met?: docker compose ls | grep -ioE -q "lhm.*running\(3\)"
+      meet: docker compose up -d
+  - custom:
+      name: Waiting for DBs to be operational
+      met?: ./scripts/helpers/wait-for-dbs.sh
+      meet: ":"
 
 commands:
   unit: bundle exec rake unit
   int: bundle exec rake integration
   test: bundle exec rake unit && bundle exec rake integration
   cov: rm -rf coverage; COV=1 bundle exec rake unit && bundle exec rake integration; open coverage/index.html
+  logs:
+    desc: "See the DB logs (ctrl-c + ctrl-c to exit)"
+    run: docker-compose logs -f

data/docker-compose.yml

@@ -0,0 +1,46 @@
+services:
+#  Writer
+  mysql-1:
+    container_name: mysql-1
+    image: percona:5.7
+    command:
+      --server-id=1
+      --log-slave-updates=ON
+      --gtid-mode=ON
+      --enforce-gtid-consistency=ON
+      --read-only=OFF
+    hostname: 'mysql-1'
+    volumes:
+      - ./scripts/mysql/writer:/docker-entrypoint-initdb.d
+    environment:
+      MYSQL_ROOT_PASSWORD: password
+      MYSQL_HOST: mysql-1
+    ports:
+      - "33006:3306"
+#  Reader
+  mysql-2:
+    container_name: mysql-2
+    image: percona:5.7
+    command:
+      --server-id=2
+      --log-slave-updates=ON
+      --gtid-mode=ON
+      --enforce-gtid-consistency=ON
+      --read-only=ON
+    hostname: 'mysql-2'
+    volumes:
+      - ./scripts/mysql/reader:/docker-entrypoint-initdb.d
+    environment:
+      MYSQL_ROOT_PASSWORD: password
+      MYSQL_HOST: mysql-2
+    ports:
+      - "33007:3306"
+#  Proxysql
+  proxysql:
+    container_name: proxysql
+    image: proxysql/proxysql:2.0.11
+    volumes:
+      - ./scripts/proxysql/proxysql.cnf:/etc/proxysql.cnf
+    command: "proxysql -c /etc/proxysql.cnf -f --idle-threads"
+    ports:
+      - "33005:3306"

data/lib/lhm/atomic_switcher.rb

@@ -16,17 +16,12 @@ module Lhm
 
     attr_reader :connection
 
-    def initialize(migration, connection = nil, options = {})
+    def initialize(migration, connection = nil, options={})
       @migration = migration
       @connection = connection
       @origin = migration.origin
       @destination = migration.destination
-      @retry_helper = SqlRetry.new(
-        @connection,
-        {
-          log_prefix: "AtomicSwitcher"
-        }.merge!(options.fetch(:retriable, {}))
-      )
+      @retry_options = options[:retriable] || {}
     end
 
     def atomic_switch
@@ -36,7 +31,7 @@ module Lhm
 
     def validate
       unless @connection.data_source_exists?(@origin.name) &&
-             @connection.data_source_exists?(@destination.name)
+        @connection.data_source_exists?(@destination.name)
         error "`#{ @origin.name }` and `#{ @destination.name }` must exist"
       end
     end
@@ -44,9 +39,7 @@ module Lhm
     private
 
     def execute
-      @retry_helper.with_retries do |retriable_connection|
-        retriable_connection.execute atomic_switch
-      end
+      @connection.execute(atomic_switch, @retry_options)
     end
   end
 end

data/lib/lhm/chunk_insert.rb

@@ -7,18 +7,11 @@ module Lhm
       @connection = connection
       @lowest = lowest
       @highest = highest
-      @retry_helper = SqlRetry.new(
-        @connection,
-        {
-          log_prefix: "Chunker Insert"
-        }.merge!(options.fetch(:retriable, {}))
-      )
+      @retry_options = options[:retriable] || {}
     end
 
     def insert_and_return_count_of_rows_created
-      @retry_helper.with_retries do |retriable_connection|
-        retriable_connection.update sql
-      end
+      @connection.update(sql, @retry_options)
     end
 
     def sql

data/lib/lhm/chunker.rb

@@ -28,11 +28,12 @@ module Lhm
       @start = @chunk_finder.start
       @limit = @chunk_finder.limit
       @printer = options[:printer] || Printer::Percentage.new
+      @retry_options = options[:retriable] || {}
       @retry_helper = SqlRetry.new(
         @connection,
         {
           log_prefix: "Chunker"
-        }.merge!(options.fetch(:retriable, {}))
+        }.merge!(@retry_options)
       )
     end
 
@@ -46,7 +47,7 @@ module Lhm
         top = upper_id(@next_to_insert, stride)
         verify_can_run
 
-        affected_rows = ChunkInsert.new(@migration, @connection, bottom, top, @options).insert_and_return_count_of_rows_created
+        affected_rows = ChunkInsert.new(@migration, @connection, bottom, top, @retry_options).insert_and_return_count_of_rows_created
         expected_rows = top - bottom + 1
 
         # Only log the chunker progress every 5 minutes instead of every iteration
@@ -78,7 +79,7 @@ module Lhm
     private
 
     def raise_on_non_pk_duplicate_warning
-      @connection.query("show warnings").each do |level, code, message|
+      @connection.execute("show warnings", @retry_options).each do |level, code, message|
         unless message.match?(/Duplicate entry .+ for key 'PRIMARY'/)
           m = "Unexpected warning found for inserted row: #{message}"
           Lhm.logger.warn(m)
@@ -93,16 +94,14 @@ module Lhm
 
     def verify_can_run
       return unless @verifier
-      @retry_helper.with_retries do |retriable_connection|
+      @retry_helper.with_retries(@retry_options) do |retriable_connection|
         raise "Verification failed, aborting early" if !@verifier.call(retriable_connection)
       end
     end
 
     def upper_id(next_id, stride)
       sql = "select id from `#{ @migration.origin_name }` where id >= #{ next_id } order by id limit 1 offset #{ stride - 1}"
-      top = @retry_helper.with_retries do |retriable_connection|
-        retriable_connection.select_value(sql)
-      end
+      top = @connection.select_value(sql, @retry_options)
 
       [top ? top.to_i : @limit, @limit].min
     end

data/lib/lhm/cleanup/current.rb

@@ -4,17 +4,12 @@ require 'lhm/sql_retry'
 module Lhm
   module Cleanup
     class Current
-      def initialize(run, origin_table_name, connection, options = {})
+      def initialize(run, origin_table_name, connection, options={})
         @run = run
         @table_name = TableName.new(origin_table_name)
         @connection = connection
         @ddls = []
-        @retry_helper = SqlRetry.new(
-          @connection,
-          {
-            log_prefix: "Cleanup::Current"
-          }.merge!(options.fetch(:retriable, {}))
-        )
+        @retry_config = options[:retriable] || {}
       end
 
       attr_reader :run, :connection, :ddls
@@ -59,9 +54,7 @@ module Lhm
 
       def execute_ddls
         ddls.each do |ddl|
-          @retry_helper.with_retries do |retriable_connection|
-            retriable_connection.execute(ddl)
-          end
+            @connection.execute(ddl, @retry_config)
         end
         Lhm.logger.info("Dropped triggers on #{@lhm_triggers_for_origin.join(', ')}")
         Lhm.logger.info("Dropped tables #{@lhm_triggers_for_origin.join(', ')}")

data/lib/lhm/connection.rb

@@ -0,0 +1,61 @@
+require 'delegate'
+
+module Lhm
+  class Connection < SimpleDelegator
+
+    # Lhm::Connection inherits from SingleDelegator. It will forward any unknown method calls to the ActiveRecord
+    # connection.
+    alias connection __getobj__
+    alias connection= __setobj__
+
+    def initialize(connection:, default_log_prefix: nil, retry_options: {})
+      @default_log_prefix = default_log_prefix
+      @retry_options = retry_options || default_retry_config
+      @sql_retry = Lhm::SqlRetry.new(
+        connection,
+        retry_options,
+      )
+
+      # Creates delegation for the ActiveRecord Connection
+      super(connection)
+    end
+
+    def execute(query, retry_options = {})
+      exec_with_retries(:execute, query, retry_options)
+    end
+
+    def update(query, retry_options = {})
+      exec_with_retries(:update, query, retry_options)
+    end
+
+    def select_value(query, retry_options = {})
+      exec_with_retries(:select_value, query, retry_options)
+    end
+
+    private
+
+    def exec_with_retries(method, sql, retry_options = {})
+      retry_options[:log_prefix] ||= file
+      @sql_retry.with_retries(retry_options) do |conn|
+        conn.public_send(method, sql)
+      end
+    end
+
+    # Returns camelized file name of caller (e.g. chunk_insert.rb -> ChunkInsert)
+    def file
+      # Find calling file and extract name
+      /[\/]*(\w+).rb:\d+:in/.match(relevant_caller)
+      name = $1&.camelize || "Connection"
+      "#{name}"
+    end
+
+    def relevant_caller
+      lhm_stack = caller.select { |x| x.include?("/lhm") }
+      first_candidate_index = lhm_stack.find_index {|line| !line.include?(__FILE__)}
+
+      # Find the file that called the `#execute` (fallbacks to current file)
+      return lhm_stack.first unless first_candidate_index
+      lhm_stack.at(first_candidate_index)
+    end
+  end
+end

data/lib/lhm/entangler.rb

@@ -4,6 +4,7 @@
 require 'lhm/command'
 require 'lhm/sql_helper'
 require 'lhm/sql_retry'
+require 'lhm/connection'
 
 module Lhm
   class Entangler
@@ -19,12 +20,7 @@ module Lhm
       @origin = migration.origin
       @destination = migration.destination
       @connection = connection
-      @retry_helper = SqlRetry.new(
-        @connection,
-        {
-          log_prefix: "Entangler"
-        }.merge!(options.fetch(:retriable, {}))
-      )
+      @retry_options = options[:retriable] || {}
     end
 
     def entangle
@@ -90,18 +86,14 @@ module Lhm
 
     def before
       entangle.each do |stmt|
-        @retry_helper.with_retries do |retriable_connection|
-          retriable_connection.execute(stmt)
-        end
+        @connection.execute(stmt, @retry_options)
       end
       Lhm.logger.info("Created triggers on #{@origin.name}")
     end
 
     def after
       untangle.each do |stmt|
-        @retry_helper.with_retries do |retriable_connection|
-          retriable_connection.execute(stmt)
-        end
+        @connection.execute(stmt, @retry_options)
       end
       Lhm.logger.info("Dropped triggers on #{@origin.name}")
     end

data/lib/lhm/invoker.rb

@@ -56,7 +56,7 @@ module Lhm
         Chunker.new(migration, @connection, options).run
         raise "Required triggers do not exist" unless triggers_still_exist?(@connection, entangler)
         if options[:atomic_switch]
-          AtomicSwitcher.new(migration, @connection, options).run
+          AtomicSwitcher.new(migration, @connection).run
         else
           LockedSwitcher.new(migration, @connection).run
         end

data/lib/lhm/sql_retry.rb

@@ -17,20 +17,26 @@ module Lhm
   class SqlRetry
     def initialize(connection, options = {})
       @connection = connection
-      @log_prefix = options.delete(:log_prefix)
-      @retry_config = default_retry_config.dup.merge!(options)
+      @global_retry_config = default_retry_config.dup.merge!(options)
     end
 
-    def with_retries
-      Retriable.retriable(retry_config) do
+    def with_retries(retry_config = {})
+      cnf = @global_retry_config.dup.merge!(retry_config)
+      @log_prefix = cnf.delete(:log_prefix) || "SQL Retry"
+      Retriable.retriable(cnf) do
         yield(@connection)
       end
     end
 
-    attr_reader :retry_config
+    attr_reader :global_retry_config
 
     private
 
+    def log_with_prefix(message, level = :info)
+      message.prepend("[#{@log_prefix}] ") if @log_prefix
+      Lhm.logger.send(level, message)
+    end
+
     # For a full list of configuration options see https://github.com/kamui/retriable
     def default_retry_config
       {
@@ -52,8 +58,7 @@ module Lhm
         max_elapsed_time: Float::INFINITY, # max total time in seconds that code is allowed to keep being retried
         on_retry: Proc.new do |exception, try_number, total_elapsed_time, next_interval|
           log = "#{exception.class}: '#{exception.message}' - #{try_number} tries in #{total_elapsed_time} seconds and #{next_interval} seconds until the next try."
-          log.prepend("[#{@log_prefix}] ") if @log_prefix
-          Lhm.logger.info(log)
+          log_with_prefix(log, :info)
         end
       }.freeze
     end

data/lib/lhm/version.rb

@@ -2,5 +2,5 @@
 # Schmidt
 
 module Lhm
-  VERSION = '3.4.2'
+  VERSION = '3.5.0'
 end

data/lib/lhm.rb

@@ -8,6 +8,7 @@ require 'lhm/throttler'
 require 'lhm/version'
 require 'lhm/cleanup/current'
 require 'lhm/sql_retry'
+require 'lhm/connection'
 require 'lhm/test_support'
 require 'lhm/railtie' if defined?(Rails::Railtie)
 require 'logger'
@@ -82,7 +83,7 @@ module Lhm
   end
 
   def setup(connection)
-    @@connection = connection
+    @@connection = Lhm::Connection.new(connection: connection)
   end
 
   def connection

data/scripts/helpers/wait-for-dbs.sh

@@ -0,0 +1,21 @@
+#!/bin/bash
+# Wait for writer
+echo "Waiting for MySQL-1: "
+while ! (mysqladmin ping --host="127.0.0.1" --port=33006 --user=root --password=password --silent 2> /dev/null); do
+  echo -ne "."
+  sleep 1
+done
+# Wait for reader
+echo "Waiting for MySQL-2: "
+while ! (mysqladmin ping --host="127.0.0.1" --port=33007 --user=root --password=password --silent 2> /dev/null); do
+  echo -ne "."
+  sleep 1
+done
+# Wait for proxysql
+echo "Waiting for ProxySQL:"
+while ! (mysqladmin ping --host="127.0.0.1" --port=33005 --user=root --password=password --silent 2> /dev/null); do
+  echo -ne "."
+  sleep 1
+done
+
+echo "All DBs are ready"

data/scripts/mysql/reader/create_replication.sql

@@ -0,0 +1,10 @@
+STOP SLAVE;
+CHANGE MASTER TO
+MASTER_HOST='mysql-1',
+MASTER_AUTO_POSITION=1,
+MASTER_USER='replication',
+MASTER_PASSWORD='password',
+MASTER_CONNECT_RETRY=1,
+MASTER_RETRY_COUNT=300; -- 5 minutes
+
+start slave;

data/scripts/mysql/writer/create_test_db.sql

@@ -0,0 +1 @@
+CREATE DATABASE test;

data/scripts/mysql/writer/create_users.sql

@@ -0,0 +1,3 @@
+# Creates replication user in Writer
+CREATE USER IF NOT EXISTS 'replication'@'%' IDENTIFIED BY 'password';
+GRANT REPLICATION SLAVE ON *.* TO' replication'@'%' IDENTIFIED BY 'password';

data/scripts/proxysql/proxysql.cnf

@@ -0,0 +1,117 @@
+#file proxysql.cfg
+
+datadir="/var/lib/proxysql"
+restart_on_missing_heartbeats=999999
+query_parser_token_delimiters=","
+query_parser_key_value_delimiters=":"
+unit_of_work_identifiers="consistent_read_id"
+
+admin_variables=
+{
+  mysql_ifaces="0.0.0.0:6032"
+  admin_credentials="admin:password;remote-admin:password"
+}
+
+mysql_servers =
+(
+  {
+    address="mysql-1"
+    port=3306
+    hostgroup=0
+    max_connections=200
+  },
+  {
+    address="mysql-2"
+    port=3306
+    hostgroup=1
+    max_connections=200
+  }
+)
+
+mysql_variables=
+{
+  session_idle_ms=1
+    auto_increment_delay_multiplex=0
+
+    threads=8
+    max_connections=100000
+    interfaces="0.0.0.0:3306"
+    server_version="5.7.18-proxysql"
+    connect_timeout_server=10000
+    connect_timeout_server_max=10000
+    connect_retries_on_failure=0
+    default_charset="utf8mb4"
+    free_connections_pct=100
+    connection_warming=true
+    max_allowed_packet=16777216
+    monitor_enabled=false
+    query_retries_on_failure=0
+    shun_on_failures=999999
+    shun_recovery_time_sec=0
+    kill_backend_connection_when_disconnect=false
+    stats_time_backend_query=false
+    stats_time_query_processor=false
+    max_stmts_per_connection=5
+    default_max_latency_ms=999999
+    wait_timeout=1800000
+    eventslog_format=3
+    log_multiplexing_disabled=true
+    log_unhealthy_connections=false
+}
+
+# defines all the MySQL users
+mysql_users:
+(
+  {
+    username = "root"
+    password = "password"
+    default_hostgroup = 0
+    max_connections=1000
+    active = 1
+  },
+  {
+    username = "writer"
+    password = "password"
+    default_hostgroup = 0
+    max_connections=50000
+    active = 1
+    transaction_persistent=1
+  },
+  {
+    username = "reader"
+    password = "password"
+    default_hostgroup = 1
+    max_connections=50000
+    active = 1
+    transaction_persistent=1
+  }
+)
+
+#defines MySQL Query Rules
+mysql_query_rules:
+(
+  {
+      rule_id = 1
+      active = 1
+      match_digest = "@@SESSION"
+      multiplex = 2
+    },
+    {
+      rule_id = 2
+      active = 1
+      match_digest = "@@global\.server_id"
+      multiplex = 2
+    },
+    {
+      rule_id = 3
+      active = 1
+      match_digest = "@@global\.hostname"
+      multiplex = 2
+    },
+    {
+      rule_id = 4
+      active = 1
+      match_pattern = "maintenance:lhm"
+      destination_hostgroup = 0
+    }
+)

data/spec/integration/atomic_switcher_spec.rb

@@ -6,6 +6,7 @@ require File.expand_path(File.dirname(__FILE__)) + '/integration_helper'
 require 'lhm/table'
 require 'lhm/migration'
 require 'lhm/atomic_switcher'
+require 'lhm/connection'
 
 describe Lhm::AtomicSwitcher do
   include IntegrationHelper
@@ -29,9 +30,11 @@ describe Lhm::AtomicSwitcher do
     end
 
     it 'should retry and log on lock wait timeouts' do
-      connection = mock()
-      connection.stubs(:data_source_exists?).returns(true)
-      connection.stubs(:execute).raises(ActiveRecord::StatementInvalid, 'Lock wait timeout exceeded; try restarting transaction.').then.returns(true)
+      ar_connection = mock()
+      ar_connection.stubs(:data_source_exists?).returns(true)
+      ar_connection.stubs(:execute).raises(ActiveRecord::StatementInvalid, 'Lock wait timeout exceeded; try restarting transaction.').then.returns(true)
+
+      connection = Lhm::Connection.new(connection: ar_connection)
 
       switcher = Lhm::AtomicSwitcher.new(@migration, connection, retriable: {base_interval: 0})
 
@@ -40,13 +43,16 @@ describe Lhm::AtomicSwitcher do
       log_messages = @logs.string.split("\n")
       assert_equal(2, log_messages.length)
       assert log_messages[0].include? "Starting run of class=Lhm::AtomicSwitcher"
+      # On failure of this assertion, check for Lhm::Connection#file
       assert log_messages[1].include? "[AtomicSwitcher] ActiveRecord::StatementInvalid: 'Lock wait timeout exceeded; try restarting transaction.' - 1 tries"
     end
 
     it 'should give up on lock wait timeouts after a configured number of tries' do
-      connection = mock()
-      connection.stubs(:data_source_exists?).returns(true)
-      connection.stubs(:execute).twice.raises(ActiveRecord::StatementInvalid, 'Lock wait timeout exceeded; try restarting transaction.')
+      ar_connection = mock()
+      ar_connection.stubs(:data_source_exists?).returns(true)
+      ar_connection.stubs(:execute).twice.raises(ActiveRecord::StatementInvalid, 'Lock wait timeout exceeded; try restarting transaction.')
+
+      connection = Lhm::Connection.new(connection: ar_connection)
 
       switcher = Lhm::AtomicSwitcher.new(@migration, connection, retriable: {tries: 2, base_interval: 0})
 
@@ -62,8 +68,10 @@ describe Lhm::AtomicSwitcher do
     end
 
     it "should raise when destination doesn't exist" do
-      connection = mock()
-      connection.stubs(:data_source_exists?).returns(false)
+      ar_connection = mock()
+      ar_connection.stubs(:data_source_exists?).returns(false)
+
+      connection = Lhm::Connection.new(connection: ar_connection)
 
       switcher = Lhm::AtomicSwitcher.new(@migration, connection)
 

data/spec/integration/database.yml

@@ -0,0 +1,15 @@
+master:
+  host: mysql-1
+  user: root
+  password: password
+  port: 33006
+slave:
+  host: mysql-2
+  user: root
+  password: password
+  port: 33007
+proxysql:
+  host: proxysql
+  user: root
+  password: password
+  port: 33005

data/spec/integration/integration_helper.rb

@@ -41,7 +41,6 @@ module IntegrationHelper
       $db_config['master']['port'],
       $db_config['master']['user'],
       $db_config['master']['password'],
-      $db_config['master']['socket']
     )
   end
 
@@ -51,12 +50,11 @@ module IntegrationHelper
       $db_config['slave']['port'],
       $db_config['slave']['user'],
       $db_config['slave']['password'],
-      $db_config['slave']['socket']
     )
   end
 
-  def connect!(hostname, port, user, password, socket)
-    adapter = ar_conn(hostname, port, user, password, socket)
+  def connect!(hostname, port, user, password)
+    adapter = Lhm::Connection.new(connection: ar_conn(hostname, port, user, password))
     Lhm.setup(adapter)
     unless defined?(@@cleaned_up)
       Lhm.cleanup(true)
@@ -65,14 +63,13 @@ module IntegrationHelper
     @connection = adapter
   end
 
-  def ar_conn(host, port, user, password, socket)
+  def ar_conn(host, port, user, password)
     ActiveRecord::Base.establish_connection(
       :adapter  => 'mysql2',
       :host     => host,
       :username => user,
       :port     => port,
       :password => password,
-      :socket   => socket,
       :database => $db_name
     )
     ActiveRecord::Base.connection

data/spec/integration/sql_retry/lock_wait_timeout_test_helper.rb

@@ -77,14 +77,17 @@ class LockWaitTimeoutTestHelper
   attr_reader :main_conn, :lock_duration, :innodb_lock_wait_timeout
 
   def new_mysql_connection
-    Mysql2::Client.new(
+    client = Mysql2::Client.new(
       host: '127.0.0.1',
-      database: test_db_name,
       username: db_config['master']['user'],
       password: db_config['master']['password'],
-      port: db_config['master']['port'],
-      socket: db_config['master']['socket']
+      port: db_config['master']['port']
     )
+
+    # For some reasons sometimes the database does not exist
+    client.query("CREATE DATABASE IF NOT EXISTS #{test_db_name}")
+    client.select_db(test_db_name)
+    client
   end
 
   def test_db_name

data/spec/unit/chunker_spec.rb

@@ -7,6 +7,7 @@ require 'lhm/table'
 require 'lhm/migration'
 require 'lhm/chunker'
 require 'lhm/throttler'
+require 'lhm/connection'
 
 describe Lhm::Chunker do
   include UnitHelper
@@ -37,11 +38,11 @@ describe Lhm::Chunker do
         5
       end
 
-      @connection.expects(:select_value).with(regexp_matches(/where id >= 1 order by id limit 1 offset 4/)).returns(7)
-      @connection.expects(:select_value).with(regexp_matches(/where id >= 8 order by id limit 1 offset 4/)).returns(21)
-      @connection.expects(:update).with(regexp_matches(/between 1 and 7/)).returns(2)
-      @connection.expects(:update).with(regexp_matches(/between 8 and 10/)).returns(2)
-      @connection.expects(:query).twice.with(regexp_matches(/show warnings/)).returns([])
+      @connection.expects(:select_value).with(regexp_matches(/where id >= 1 order by id limit 1 offset 4/),{}).returns(7)
+      @connection.expects(:select_value).with(regexp_matches(/where id >= 8 order by id limit 1 offset 4/),{}).returns(21)
+      @connection.expects(:update).with(regexp_matches(/between 1 and 7/),{}).returns(2)
+      @connection.expects(:update).with(regexp_matches(/between 8 and 10/),{}).returns(2)
+      @connection.expects(:execute).twice.with(regexp_matches(/show warnings/),{}).returns([])
 
       @chunker.run
     end
@@ -52,17 +53,17 @@ describe Lhm::Chunker do
         2
       end
 
-      @connection.expects(:select_value).with(regexp_matches(/where id >= 1 order by id limit 1 offset 1/)).returns(2)
-      @connection.expects(:select_value).with(regexp_matches(/where id >= 3 order by id limit 1 offset 1/)).returns(4)
-      @connection.expects(:select_value).with(regexp_matches(/where id >= 5 order by id limit 1 offset 1/)).returns(6)
-      @connection.expects(:select_value).with(regexp_matches(/where id >= 7 order by id limit 1 offset 1/)).returns(8)
-      @connection.expects(:select_value).with(regexp_matches(/where id >= 9 order by id limit 1 offset 1/)).returns(10)
+      @connection.expects(:select_value).with(regexp_matches(/where id >= 1 order by id limit 1 offset 1/),{}).returns(2)
+      @connection.expects(:select_value).with(regexp_matches(/where id >= 3 order by id limit 1 offset 1/),{}).returns(4)
+      @connection.expects(:select_value).with(regexp_matches(/where id >= 5 order by id limit 1 offset 1/),{}).returns(6)
+      @connection.expects(:select_value).with(regexp_matches(/where id >= 7 order by id limit 1 offset 1/),{}).returns(8)
+      @connection.expects(:select_value).with(regexp_matches(/where id >= 9 order by id limit 1 offset 1/),{}).returns(10)
 
-      @connection.expects(:update).with(regexp_matches(/between 1 and 2/)).returns(2)
-      @connection.expects(:update).with(regexp_matches(/between 3 and 4/)).returns(2)
-      @connection.expects(:update).with(regexp_matches(/between 5 and 6/)).returns(2)
-      @connection.expects(:update).with(regexp_matches(/between 7 and 8/)).returns(2)
-      @connection.expects(:update).with(regexp_matches(/between 9 and 10/)).returns(2)
+      @connection.expects(:update).with(regexp_matches(/between 1 and 2/),{}).returns(2)
+      @connection.expects(:update).with(regexp_matches(/between 3 and 4/),{}).returns(2)
+      @connection.expects(:update).with(regexp_matches(/between 5 and 6/),{}).returns(2)
+      @connection.expects(:update).with(regexp_matches(/between 7 and 8/),{}).returns(2)
+      @connection.expects(:update).with(regexp_matches(/between 9 and 10/),{}).returns(2)
 
       @chunker.run
     end
@@ -79,17 +80,17 @@ describe Lhm::Chunker do
         end
       end
 
-      @connection.expects(:select_value).with(regexp_matches(/where id >= 1 order by id limit 1 offset 1/)).returns(2)
-      @connection.expects(:select_value).with(regexp_matches(/where id >= 3 order by id limit 1 offset 2/)).returns(5)
-      @connection.expects(:select_value).with(regexp_matches(/where id >= 6 order by id limit 1 offset 2/)).returns(8)
-      @connection.expects(:select_value).with(regexp_matches(/where id >= 9 order by id limit 1 offset 2/)).returns(nil)
+      @connection.expects(:select_value).with(regexp_matches(/where id >= 1 order by id limit 1 offset 1/),{}).returns(2)
+      @connection.expects(:select_value).with(regexp_matches(/where id >= 3 order by id limit 1 offset 2/),{}).returns(5)
+      @connection.expects(:select_value).with(regexp_matches(/where id >= 6 order by id limit 1 offset 2/),{}).returns(8)
+      @connection.expects(:select_value).with(regexp_matches(/where id >= 9 order by id limit 1 offset 2/),{}).returns(nil)
 
-      @connection.expects(:update).with(regexp_matches(/between 1 and 2/)).returns(2)
-      @connection.expects(:update).with(regexp_matches(/between 3 and 5/)).returns(2)
-      @connection.expects(:update).with(regexp_matches(/between 6 and 8/)).returns(2)
-      @connection.expects(:update).with(regexp_matches(/between 9 and 10/)).returns(2)
+      @connection.expects(:update).with(regexp_matches(/between 1 and 2/),{}).returns(2)
+      @connection.expects(:update).with(regexp_matches(/between 3 and 5/),{}).returns(2)
+      @connection.expects(:update).with(regexp_matches(/between 6 and 8/),{}).returns(2)
+      @connection.expects(:update).with(regexp_matches(/between 9 and 10/),{}).returns(2)
 
-      @connection.expects(:query).twice.with(regexp_matches(/show warnings/)).returns([])
+      @connection.expects(:execute).twice.with(regexp_matches(/show warnings/),{}).returns([])
 
       @chunker.run
     end
@@ -99,8 +100,8 @@ describe Lhm::Chunker do
                                                            :start     => 1,
                                                            :limit     => 1)
 
-      @connection.expects(:select_value).with(regexp_matches(/where id >= 1 order by id limit 1 offset 0/)).returns(nil)
-      @connection.expects(:update).with(regexp_matches(/between 1 and 1/)).returns(1)
+      @connection.expects(:select_value).with(regexp_matches(/where id >= 1 order by id limit 1 offset 0/),{}).returns(nil)
+      @connection.expects(:update).with(regexp_matches(/between 1 and 1/),{}).returns(1)
 
       @chunker.run
     end
@@ -113,17 +114,17 @@ describe Lhm::Chunker do
         2
       end
 
-      @connection.expects(:select_value).with(regexp_matches(/where id >= 2 order by id limit 1 offset 1/)).returns(3)
-      @connection.expects(:select_value).with(regexp_matches(/where id >= 4 order by id limit 1 offset 1/)).returns(5)
-      @connection.expects(:select_value).with(regexp_matches(/where id >= 6 order by id limit 1 offset 1/)).returns(7)
-      @connection.expects(:select_value).with(regexp_matches(/where id >= 8 order by id limit 1 offset 1/)).returns(9)
-      @connection.expects(:select_value).with(regexp_matches(/where id >= 10 order by id limit 1 offset 1/)).returns(nil)
+      @connection.expects(:select_value).with(regexp_matches(/where id >= 2 order by id limit 1 offset 1/),{}).returns(3)
+      @connection.expects(:select_value).with(regexp_matches(/where id >= 4 order by id limit 1 offset 1/),{}).returns(5)
+      @connection.expects(:select_value).with(regexp_matches(/where id >= 6 order by id limit 1 offset 1/),{}).returns(7)
+      @connection.expects(:select_value).with(regexp_matches(/where id >= 8 order by id limit 1 offset 1/),{}).returns(9)
+      @connection.expects(:select_value).with(regexp_matches(/where id >= 10 order by id limit 1 offset 1/),{}).returns(nil)
 
-      @connection.expects(:update).with(regexp_matches(/between 2 and 3/)).returns(2)
-      @connection.expects(:update).with(regexp_matches(/between 4 and 5/)).returns(2)
-      @connection.expects(:update).with(regexp_matches(/between 6 and 7/)).returns(2)
-      @connection.expects(:update).with(regexp_matches(/between 8 and 9/)).returns(2)
-      @connection.expects(:update).with(regexp_matches(/between 10 and 10/)).returns(1)
+      @connection.expects(:update).with(regexp_matches(/between 2 and 3/),{}).returns(2)
+      @connection.expects(:update).with(regexp_matches(/between 4 and 5/),{}).returns(2)
+      @connection.expects(:update).with(regexp_matches(/between 6 and 7/),{}).returns(2)
+      @connection.expects(:update).with(regexp_matches(/between 8 and 9/),{}).returns(2)
+      @connection.expects(:update).with(regexp_matches(/between 10 and 10/),{}).returns(1)
 
       @chunker.run
     end
@@ -137,9 +138,9 @@ describe Lhm::Chunker do
         2
       end
 
-      @connection.expects(:select_value).with(regexp_matches(/where id >= 1 order by id limit 1 offset 1/)).returns(2)
-      @connection.expects(:update).with(regexp_matches(/where \(foo.created_at > '2013-07-10' or foo.baz = 'quux'\) and `foo`/)).returns(1)
-      @connection.expects(:query).with(regexp_matches(/show warnings/)).returns([])
+      @connection.expects(:select_value).with(regexp_matches(/where id >= 1 order by id limit 1 offset 1/),{}).returns(2)
+      @connection.expects(:update).with(regexp_matches(/where \(foo.created_at > '2013-07-10' or foo.baz = 'quux'\) and `foo`/),{}).returns(1)
+      @connection.expects(:execute).with(regexp_matches(/show warnings/),{}).returns([])
 
       def @migration.conditions
         "where foo.created_at > '2013-07-10' or foo.baz = 'quux'"
@@ -157,9 +158,9 @@ describe Lhm::Chunker do
         2
       end
 
-      @connection.expects(:select_value).with(regexp_matches(/where id >= 1 order by id limit 1 offset 1/)).returns(2)
-      @connection.expects(:update).with(regexp_matches(/inner join bar on foo.id = bar.foo_id and/)).returns(1)
-      @connection.expects(:query).with(regexp_matches(/show warnings/)).returns([])
+      @connection.expects(:select_value).with(regexp_matches(/where id >= 1 order by id limit 1 offset 1/),{}).returns(2)
+      @connection.expects(:update).with(regexp_matches(/inner join bar on foo.id = bar.foo_id and/),{}).returns(1)
+      @connection.expects(:execute).with(regexp_matches(/show warnings/),{}).returns([])
 
       def @migration.conditions
         'inner join bar on foo.id = bar.foo_id'

data/spec/unit/connection_spec.rb

@@ -0,0 +1,68 @@
+require 'lhm/connection'
+
+describe Lhm::Connection do
+
+  LOCK_WAIT = ActiveRecord::StatementInvalid.new('Lock wait timeout exceeded; try restarting transaction.')
+
+  before(:each) do
+    @logs = StringIO.new
+    Lhm.logger = Logger.new(@logs)
+  end
+
+  it "Should find use calling file as prefix" do
+    ar_connection = mock()
+    ar_connection.stubs(:execute).raises(LOCK_WAIT).then.returns(true)
+
+    connection = Lhm::Connection.new(connection: ar_connection)
+
+    connection.execute("SHOW TABLES", { base_interval: 0 })
+
+    log_messages = @logs.string.split("\n")
+    assert_equal(1, log_messages.length)
+    assert log_messages.first.include?("[ConnectionSpec]")
+  end
+
+  it "#execute should be retried" do
+    ar_connection = mock()
+    ar_connection.stubs(:execute).raises(LOCK_WAIT)
+                 .then.raises(LOCK_WAIT)
+                 .then.returns(true)
+
+    connection = Lhm::Connection.new(connection: ar_connection)
+
+    connection.execute("SHOW TABLES", { base_interval: 0, tries: 3 })
+
+    log_messages = @logs.string.split("\n")
+    assert_equal(2, log_messages.length)
+  end
+
+  it "#update should be retried" do
+    ar_connection = mock()
+    ar_connection.stubs(:update).raises(LOCK_WAIT)
+                 .then.raises(LOCK_WAIT)
+                 .then.returns(1)
+
+    connection = Lhm::Connection.new(connection: ar_connection)
+
+    val = connection.update("SHOW TABLES", { base_interval: 0, tries: 3 })
+
+    log_messages = @logs.string.split("\n")
+    assert_equal val, 1
+    assert_equal(2, log_messages.length)
+  end
+
+  it "#select_value should be retried" do
+    ar_connection = mock()
+    ar_connection.stubs(:select_value).raises(LOCK_WAIT)
+                 .then.raises(LOCK_WAIT)
+                 .then.returns("dummy")
+
+    connection = Lhm::Connection.new(connection: ar_connection)
+
+    val = connection.select_value("SHOW TABLES", { base_interval: 0, tries: 3 })
+
+    log_messages = @logs.string.split("\n")
+    assert_equal val, "dummy"
+    assert_equal(2, log_messages.length)
+  end
+end

data/spec/unit/entangler_spec.rb

@@ -6,6 +6,7 @@ require File.expand_path(File.dirname(__FILE__)) + '/unit_helper'
 require 'lhm/table'
 require 'lhm/migration'
 require 'lhm/entangler'
+require 'lhm/connection'
 
 describe Lhm::Entangler do
   include UnitHelper
@@ -60,33 +61,43 @@ describe Lhm::Entangler do
     end
 
     it 'should retry trigger creation when it hits a lock wait timeout' do
-      connection = mock()
       tries = 1
+      ar_connection = mock()
+      ar_connection.expects(:execute).times(tries).raises(Mysql2::Error, 'Lock wait timeout exceeded; try restarting transaction')
+
+      connection = Lhm::Connection.new(connection: ar_connection)
+
       @entangler = Lhm::Entangler.new(@migration, connection, retriable: {base_interval: 0, tries: tries})
-      connection.expects(:execute).times(tries).raises(Mysql2::Error, 'Lock wait timeout exceeded; try restarting transaction')
 
       assert_raises(Mysql2::Error) { @entangler.before }
     end
 
     it 'should not retry trigger creation with other mysql errors' do
-      connection = mock()
-      connection.expects(:execute).once.raises(Mysql2::Error, 'The MySQL server is running with the --read-only option so it cannot execute this statement.')
+      ar_connection = mock()
+      ar_connection.expects(:execute).once.raises(Mysql2::Error, 'The MySQL server is running with the --read-only option so it cannot execute this statement.')
+      connection = Lhm::Connection.new(connection: ar_connection)
 
       @entangler = Lhm::Entangler.new(@migration, connection, retriable: {base_interval: 0})
       assert_raises(Mysql2::Error) { @entangler.before }
     end
 
     it 'should succesfully finish after retrying' do
-      connection = mock()
-      connection.stubs(:execute).raises(Mysql2::Error, 'Lock wait timeout exceeded; try restarting transaction').then.returns(true)
+      ar_connection = mock()
+      ar_connection.stubs(:execute).raises(Mysql2::Error, 'Lock wait timeout exceeded; try restarting transaction').then.returns(true)
+
+      connection = Lhm::Connection.new(connection: ar_connection)
+
       @entangler = Lhm::Entangler.new(@migration, connection, retriable: {base_interval: 0})
 
       assert @entangler.before
     end
 
     it 'should retry as many times as specified by configuration' do
-      connection = mock()
-      connection.expects(:execute).times(5).raises(Mysql2::Error, 'Lock wait timeout exceeded; try restarting transaction')
+      ar_connection = mock()
+      ar_connection.expects(:execute).times(5).raises(Mysql2::Error, 'Lock wait timeout exceeded; try restarting transaction')
+
+      connection = Lhm::Connection.new(connection: ar_connection)
+
       @entangler = Lhm::Entangler.new(@migration, connection, retriable: {tries: 5, base_interval: 0})
 
       assert_raises(Mysql2::Error) { @entangler.before }

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: lhm-shopify
 version: !ruby/object:Gem::Version
-  version: 3.4.2
+  version: 3.5.0
 platform: ruby
 authors:
 - SoundCloud
@@ -12,7 +12,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2021-09-28 00:00:00.000000000 Z
+date: 2021-12-07 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: retriable
@@ -129,10 +129,8 @@ files:
 - LICENSE
 - README.md
 - Rakefile
-- bin/.gitkeep
-- dbdeployer/config.json
-- dbdeployer/install.sh
 - dev.yml
+- docker-compose.yml
 - gemfiles/ar-2.3_mysql.gemfile
 - gemfiles/ar-3.2_mysql.gemfile
 - gemfiles/ar-3.2_mysql2.gemfile
@@ -149,6 +147,7 @@ files:
 - lib/lhm/chunker.rb
 - lib/lhm/cleanup/current.rb
 - lib/lhm/command.rb
+- lib/lhm/connection.rb
 - lib/lhm/entangler.rb
 - lib/lhm/intersection.rb
 - lib/lhm/invoker.rb
@@ -168,6 +167,11 @@ files:
 - lib/lhm/throttler/time.rb
 - lib/lhm/timestamp.rb
 - lib/lhm/version.rb
+- scripts/helpers/wait-for-dbs.sh
+- scripts/mysql/reader/create_replication.sql
+- scripts/mysql/writer/create_test_db.sql
+- scripts/mysql/writer/create_users.sql
+- scripts/proxysql/proxysql.cnf
 - shipit.rubygems.yml
 - spec/.lhm.example
 - spec/README.md
@@ -188,6 +192,7 @@ files:
 - spec/integration/chunk_insert_spec.rb
 - spec/integration/chunker_spec.rb
 - spec/integration/cleanup_spec.rb
+- spec/integration/database.yml
 - spec/integration/entangler_spec.rb
 - spec/integration/integration_helper.rb
 - spec/integration/invoker_spec.rb
@@ -202,6 +207,7 @@ files:
 - spec/unit/chunk_finder_spec.rb
 - spec/unit/chunk_insert_spec.rb
 - spec/unit/chunker_spec.rb
+- spec/unit/connection_spec.rb
 - spec/unit/entangler_spec.rb
 - spec/unit/intersection_spec.rb
 - spec/unit/lhm_spec.rb

data/dbdeployer/config.json

@@ -1,32 +0,0 @@
-{
- 	"version": "1.8.0",
- 	"sandbox-home": "./dbdeployer/sandboxes",
- 	"sandbox-binary": "./dbdeployer/binaries",
- 	"use-sandbox-catalog": true,
- 	"master-slave-base-port": 11000,
- 	"group-replication-base-port": 12000,
- 	"group-replication-sp-base-port": 13000,
- 	"fan-in-replication-base-port": 14000,
- 	"all-masters-replication-base-port": 15000,
- 	"multiple-base-port": 16000,
- 	"group-port-delta": 125,
- 	"mysqlx-port-delta": 10000,
- 	"master-name": "master",
- 	"master-abbr": "m",
- 	"node-prefix": "node",
- 	"slave-prefix": "slave",
- 	"slave-abbr": "s",
- 	"sandbox-prefix": "msb_",
- 	"master-slave-prefix": "rsandbox_",
- 	"group-prefix": "group_msb_",
- 	"group-sp-prefix": "group_sp_msb_",
- 	"multiple-prefix": "multi_msb_",
- 	"fan-in-prefix": "fan_in_msb_",
- 	"all-masters-prefix": "all_masters_msb_",
- 	"reserved-ports": [
- 		1186,
- 		3306,
- 		33060
- 	],
- 	"timestamp": "Mon Jul 16 17:36:55 AST 2018"
- }

data/dbdeployer/install.sh

@@ -1,64 +0,0 @@
-set -e
-mkdir -p ./dbdeployer/sandboxes
-mkdir -p ./dbdeployer/binaries
-
-if [ -z "$(uname | grep Darwin)" ]; then
-  OS=linux
-  set -x
-else
-  OS=osx
-fi
-
-echo "Checking if dbdeployer is installed"
-if ! [ -x "$(command -v ./bin/dbdeployer)" ]; then
-  echo "Not installed...starting install"
-  VERSION=1.56.0
-  origin=https://github.com/datacharmer/dbdeployer/releases/download/v$VERSION
-  filename=dbdeployer-$VERSION.$OS
-  wget -q $origin/$filename.tar.gz
-  tar -xzf $filename.tar.gz
-  chmod +x $filename
-  sudo mv $filename ./bin/dbdeployer
-  rm $filename.tar.gz
-else
-  echo "Installation found!"
-fi
-
-
-echo "Checking if mysql 5.7.22 is available for dbdeployer"
-if [ -z "$(./bin/dbdeployer --config ./dbdeployer/config.json --sandbox-binary "./dbdeployer/binaries" available | grep 5.7.22)" ]; then
-  echo "Not found..."
-
-  if [ "$OS" = "linux" ]; then
-    MYSQL_FILE=mysql-5.7.22-linux-glibc2.12-x86_64.tar.gz
-  else
-    MYSQL_FILE=mysql-5.7.22-macos10.13-x86_64.tar.gz
-  fi
-
-  if [ ! -f $MYSQL_FILE ]; then
-    echo "Downloading $MYSQL_FILE...(this may take a while)"
-    wget -q "https://dev.mysql.com/get/Downloads/MySQL-5.7/$MYSQL_FILE"
-  fi
-
-  echo "Setting up..."
-  ./bin/dbdeployer unpack $MYSQL_FILE --verbosity 0 --config ./dbdeployer/config.json --sandbox-binary "./dbdeployer/binaries"
-  rm $MYSQL_FILE
-else
-  echo "mysql 5.7.22 found!"
-fi
-
-echo "Forcing new replication setup..."
-./bin/dbdeployer deploy replication 5.7.22 --nodes 2 --force --config ./dbdeployer/config.json --sandbox-binary "./dbdeployer/binaries" --sandbox-home "./dbdeployer/sandboxes"
-./bin/dbdeployer global status --config ./dbdeployer/config.json --sandbox-binary "./dbdeployer/binaries" --sandbox-home "./dbdeployer/sandboxes"
-
-echo "Setting up database.yml"
-DATABASE_YML=spec/integration/database.yml
-echo "master:" > $DATABASE_YML
-cat ./dbdeployer/sandboxes/rsandbox_5_7_22/master/my.sandbox.cnf | grep -A 4 client | tail -n 4 | sed -e 's/  * = /: /' -e 's/^/  /' >> $DATABASE_YML
-
-echo "slave:" >> $DATABASE_YML
-cat ./dbdeployer/sandboxes/rsandbox_5_7_22/node1/my.sandbox.cnf | grep -A 4 client | tail -n 4 | sed -e 's/  * = /: /' -e 's/^/  /' >> $DATABASE_YML
-
-cat $DATABASE_YML
-
-echo "You are ready to run the integration test suite..."