lhm-shopify 3.5.0 → 3.5.4

data/lhm.gemspec

@@ -1,6 +1,6 @@
-# -*- encoding: utf-8 -*-
+# frozen_string_literal: true
 
-lib = File.expand_path('../lib', __FILE__)
+lib = File.expand_path('lib', __dir__)
 $:.unshift(lib) unless $:.include?(lib)
 
 require 'lhm/version'
@@ -25,10 +25,14 @@ Gem::Specification.new do |s|
 
   s.add_dependency 'retriable', '>= 3.0.0'
 
+  s.add_development_dependency 'activerecord'
   s.add_development_dependency 'minitest'
   s.add_development_dependency 'mocha'
+  s.add_development_dependency 'after_do'
   s.add_development_dependency 'rake'
-  s.add_development_dependency 'activerecord'
   s.add_development_dependency 'mysql2'
   s.add_development_dependency 'simplecov'
+  s.add_development_dependency 'toxiproxy'
+  s.add_development_dependency 'appraisal'
+  s.add_development_dependency 'byebug'
 end

data/lib/lhm/atomic_switcher.rb

@@ -16,12 +16,13 @@ module Lhm
 
     attr_reader :connection
 
-    def initialize(migration, connection = nil, options={})
+    LOG_PREFIX = "AtomicSwitcher"
+
+    def initialize(migration, connection = nil)
       @migration = migration
       @connection = connection
       @origin = migration.origin
       @destination = migration.destination
-      @retry_options = options[:retriable] || {}
     end
 
     def atomic_switch
@@ -39,7 +40,7 @@ module Lhm
     private
 
     def execute
-      @connection.execute(atomic_switch, @retry_options)
+      @connection.execute(atomic_switch, should_retry: true, log_prefix: LOG_PREFIX)
     end
   end
 end

data/lib/lhm/chunk_insert.rb

@@ -1,17 +1,21 @@
 require 'lhm/sql_retry'
+require 'lhm/proxysql_helper'
 
 module Lhm
   class ChunkInsert
-    def initialize(migration, connection, lowest, highest, options = {})
+
+    LOG_PREFIX = "ChunkInsert"
+
+    def initialize(migration, connection, lowest, highest, retry_options = {})
       @migration = migration
       @connection = connection
       @lowest = lowest
       @highest = highest
-      @retry_options = options[:retriable] || {}
+      @retry_options = retry_options
     end
 
     def insert_and_return_count_of_rows_created
-      @connection.update(sql, @retry_options)
+      @connection.update(sql, should_retry: true, log_prefix: LOG_PREFIX)
     end
 
     def sql

data/lib/lhm/chunker.rb

@@ -13,6 +13,8 @@ module Lhm
 
     attr_reader :connection
 
+    LOG_PREFIX = "Chunker"
+
     # Copy from origin to destination in chunks of size `stride`.
     # Use the `throttler` class to sleep between each stride.
     def initialize(migration, connection = nil, options = {})
@@ -31,9 +33,7 @@ module Lhm
       @retry_options = options[:retriable] || {}
       @retry_helper = SqlRetry.new(
         @connection,
-        {
-          log_prefix: "Chunker"
-        }.merge!(@retry_options)
+        retry_options: @retry_options
       )
     end
 
@@ -79,7 +79,7 @@ module Lhm
     private
 
     def raise_on_non_pk_duplicate_warning
-      @connection.execute("show warnings", @retry_options).each do |level, code, message|
+      @connection.execute("show warnings", should_retry: true, log_prefix: LOG_PREFIX).each do |level, code, message|
         unless message.match?(/Duplicate entry .+ for key 'PRIMARY'/)
           m = "Unexpected warning found for inserted row: #{message}"
           Lhm.logger.warn(m)
@@ -94,14 +94,14 @@ module Lhm
 
     def verify_can_run
       return unless @verifier
-      @retry_helper.with_retries(@retry_options) do |retriable_connection|
+      @retry_helper.with_retries(log_prefix: LOG_PREFIX) do |retriable_connection|
         raise "Verification failed, aborting early" if !@verifier.call(retriable_connection)
       end
     end
 
     def upper_id(next_id, stride)
       sql = "select id from `#{ @migration.origin_name }` where id >= #{ next_id } order by id limit 1 offset #{ stride - 1}"
-      top = @connection.select_value(sql, @retry_options)
+      top = @connection.select_value(sql, should_retry: true, log_prefix: LOG_PREFIX)
 
       [top ? top.to_i : @limit, @limit].min
     end

data/lib/lhm/cleanup/current.rb

@@ -4,6 +4,9 @@ require 'lhm/sql_retry'
 module Lhm
   module Cleanup
     class Current
+
+      LOG_PREFIX = "Current"
+
       def initialize(run, origin_table_name, connection, options={})
         @run = run
         @table_name = TableName.new(origin_table_name)
@@ -54,7 +57,7 @@ module Lhm
 
       def execute_ddls
         ddls.each do |ddl|
-            @connection.execute(ddl, @retry_config)
+            @connection.execute(ddl, should_retry: true, log_prefix: LOG_PREFIX)
         end
         Lhm.logger.info("Dropped triggers on #{@lhm_triggers_for_origin.join(', ')}")
         Lhm.logger.info("Dropped tables #{@lhm_triggers_for_origin.join(', ')}")

data/lib/lhm/connection.rb

@@ -1,43 +1,90 @@
 require 'delegate'
+require 'forwardable'
+require 'lhm/sql_retry'
 
 module Lhm
-  class Connection < SimpleDelegator
-
     # Lhm::Connection inherits from SingleDelegator. It will forward any unknown method calls to the ActiveRecord
     # connection.
-    alias connection __getobj__
-    alias connection= __setobj__
+  class Connection < SimpleDelegator
+    extend Forwardable
+
+    # Will delegate the following function to @sql_retry object, while leaving them accessible from the Lhm::Connection
+    # object
+    def_delegators :@sql_retry, :reconnect_with_consistent_host, :reconnect_with_consistent_host=, :retry_config=
+
+    alias ar_connection __getobj__
 
-    def initialize(connection:, default_log_prefix: nil, retry_options: {})
-      @default_log_prefix = default_log_prefix
-      @retry_options = retry_options || default_retry_config
+    def initialize(connection:, options: {})
       @sql_retry = Lhm::SqlRetry.new(
         connection,
-        retry_options,
+        retry_options: options[:retriable] || {},
+        reconnect_with_consistent_host: options[:reconnect_with_consistent_host] || false
       )
 
       # Creates delegation for the ActiveRecord Connection
       super(connection)
     end
 
-    def execute(query, retry_options = {})
-      exec_with_retries(:execute, query, retry_options)
+    def ar_connection=(connection)
+      raise Lhm::Error.new("Lhm::Connection requires an active record connection to operate") if connection.nil?
+
+      @sql_retry.connection = connection
+      # Sets connection as the delegated object
+      __setobj__(connection)
     end
 
-    def update(query, retry_options = {})
-      exec_with_retries(:update, query, retry_options)
+    # ActiveRecord::Base overridden methods to incorporate custom retry logic
+    # All other methods will be delegated
+    def execute(query, should_retry: false, log_prefix: nil)
+      if should_retry
+        exec_with_retries(:execute, query, log_prefix)
+      else
+        exec(:execute, query)
+      end
     end
 
-    def select_value(query, retry_options = {})
-      exec_with_retries(:select_value, query, retry_options)
+    def update(query, should_retry: false, log_prefix: nil)
+      if should_retry
+        exec_with_retries(:update, query, log_prefix)
+      else
+        exec(:update, query)
+      end
+    end
+
+    def select_value(query, should_retry: false, log_prefix: nil)
+      if should_retry
+        exec_with_retries(:select_value, query, log_prefix)
+      else
+        exec(:select_value, query)
+      end
+    end
+
+    def select_values(query, should_retry: false, log_prefix: nil)
+      if should_retry
+        exec_with_retries(:select_values, query, log_prefix)
+      else
+        exec(:select_values, query)
+      end
+    end
+
+    def select_one(query, should_retry: false, log_prefix: nil)
+      if should_retry
+        exec_with_retries(:select_one, query, log_prefix)
+      else
+        exec(:select_one, query)
+      end
     end
 
     private
 
-    def exec_with_retries(method, sql, retry_options = {})
-      retry_options[:log_prefix] ||= file
-      @sql_retry.with_retries(retry_options) do |conn|
-        conn.public_send(method, sql)
+    def exec(method, sql)
+      ar_connection.public_send(method, Lhm::ProxySQLHelper.tagged(sql))
+    end
+
+    def exec_with_retries(method, sql, log_prefix=nil)
+      effective_log_prefix = log_prefix || file
+      @sql_retry.with_retries(log_prefix: effective_log_prefix) do |conn|
+        conn.public_send(method, Lhm::ProxySQLHelper.tagged(sql))
       end
     end
 
@@ -51,7 +98,7 @@ module Lhm
 
     def relevant_caller
       lhm_stack = caller.select { |x| x.include?("/lhm") }
-      first_candidate_index = lhm_stack.find_index {|line| !line.include?(__FILE__)}
+      first_candidate_index = lhm_stack.find_index { |line| !line.include?(__FILE__) }
 
       # Find the file that called the `#execute` (fallbacks to current file)
       return lhm_stack.first unless first_candidate_index

data/lib/lhm/entangler.rb

@@ -13,14 +13,15 @@ module Lhm
 
     attr_reader :connection
 
+    LOG_PREFIX = "Entangler"
+
     # Creates entanglement between two tables. All creates, updates and deletes
     # to origin will be repeated on the destination table.
-    def initialize(migration, connection = nil, options = {})
+    def initialize(migration, connection = nil)
       @intersection = migration.intersection
       @origin = migration.origin
       @destination = migration.destination
       @connection = connection
-      @retry_options = options[:retriable] || {}
     end
 
     def entangle
@@ -86,14 +87,14 @@ module Lhm
 
     def before
       entangle.each do |stmt|
-        @connection.execute(stmt, @retry_options)
+        @connection.execute(stmt, should_retry: true, log_prefix: LOG_PREFIX)
       end
       Lhm.logger.info("Created triggers on #{@origin.name}")
     end
 
     def after
       untangle.each do |stmt|
-        @connection.execute(stmt, @retry_options)
+        @connection.execute(stmt, should_retry: true, log_prefix: LOG_PREFIX)
       end
       Lhm.logger.info("Dropped triggers on #{@origin.name}")
     end

data/lib/lhm/invoker.rb

@@ -16,8 +16,8 @@ module Lhm
   class Invoker
     include SqlHelper
     LOCK_WAIT_TIMEOUT_DELTA = 10
-      INNODB_LOCK_WAIT_TIMEOUT_MAX=1073741824.freeze # https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_lock_wait_timeout
-      LOCK_WAIT_TIMEOUT_MAX=31536000.freeze # https://dev.mysql.com/doc/refman/5.7/en/server-system-variables.html
+    INNODB_LOCK_WAIT_TIMEOUT_MAX = 1073741824.freeze # https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_lock_wait_timeout
+    LOCK_WAIT_TIMEOUT_MAX = 31536000.freeze # https://dev.mysql.com/doc/refman/5.7/en/server-system-variables.html
 
     attr_reader :migrator, :connection
 
@@ -49,7 +49,7 @@ module Lhm
       normalize_options(options)
       set_session_lock_wait_timeouts
       migration = @migrator.run
-      entangler = Entangler.new(migration, @connection, options)
+      entangler = Entangler.new(migration, @connection)
 
       entangler.run do
         options[:verifier] ||= Proc.new { |conn| triggers_still_exist?(conn, entangler) }
@@ -90,6 +90,8 @@ module Lhm
         options[:throttler] = Lhm.throttler
       end
 
+      Lhm.connection.retry_config = options[:retriable] || {}
+
     rescue => e
       Lhm.logger.error "LHM run failed with exception=#{e.class} message=#{e.message}"
       raise

data/lib/lhm/locked_switcher.rb

@@ -22,6 +22,8 @@ module Lhm
 
     attr_reader :connection
 
+    LOG_PREFIX = "LockedSwitcher"
+
     def initialize(migration, connection = nil)
       @migration = migration
       @connection = connection

data/lib/lhm/proxysql_helper.rb

@@ -0,0 +1,10 @@
+module Lhm
+  module ProxySQLHelper
+    extend self
+    ANNOTATION = "/*maintenance:lhm*/"
+
+    def tagged(sql)
+      "#{sql} #{ANNOTATION}"
+    end
+  end
+end

data/lib/lhm/sql_retry.rb

@@ -1,5 +1,6 @@
 require 'retriable'
 require 'lhm/sql_helper'
+require 'lhm/proxysql_helper'
 
 module Lhm
   # SqlRetry standardizes the interface for retry behavior in components like
@@ -15,26 +16,123 @@ module Lhm
   # https://github.com/kamui/retriable. Additionally, a "log_prefix" option,
   # which is unique to SqlRetry can be used to prefix log output.
   class SqlRetry
-    def initialize(connection, options = {})
+    RECONNECT_SUCCESSFUL_MESSAGE = "LHM successfully reconnected to initial host:"
+    CLOUDSQL_VERSION_COMMENT = "(Google)"
+
+    MYSQL_VAR_NAMES = {
+      hostname: "@@global.hostname",
+      server_id: "@@global.server_id",
+      version_comment: "@@version_comment",
+    }
+
+    # This internal error is used to trigger retries from the parent Retriable.retriable in #with_retries
+    class ReconnectToHostSuccessful < Lhm::Error; end
+
+    def initialize(connection, retry_options: {}, reconnect_with_consistent_host: false)
       @connection = connection
-      @global_retry_config = default_retry_config.dup.merge!(options)
+      self.retry_config = retry_options
+      self.reconnect_with_consistent_host = reconnect_with_consistent_host
     end
 
-    def with_retries(retry_config = {})
-      cnf = @global_retry_config.dup.merge!(retry_config)
-      @log_prefix = cnf.delete(:log_prefix) || "SQL Retry"
-      Retriable.retriable(cnf) do
-        yield(@connection)
+    # Complete explanation of algorithm: https://github.com/Shopify/lhm/pull/112
+    def with_retries(log_prefix: nil)
+      @log_prefix = log_prefix || "" # No prefix. Just logs
+
+      Retriable.retriable(@retry_config) do
+        # Using begin -> rescue -> end for Ruby 2.4 compatibility
+        begin
+          if @reconnect_with_consistent_host
+            raise Lhm::Error.new("Could not reconnected to initial MySQL host. Aborting to avoid data-loss") unless same_host_as_initial?
+          end
+
+          yield(@connection)
+        rescue StandardError => e
+          # Not all errors should trigger a reconnect. Some errors such be raised and abort the LHM (such as reconnecting to the wrong host).
+          # The error will be raised the connection is still active (i.e. no need to reconnect) or if the connection is
+          # dead (i.e. not active) and @reconnect_with_host is false (i.e. instructed not to reconnect)
+          raise e if @connection.active? || (!@connection.active? && !@reconnect_with_consistent_host)
+          reconnect_with_host_check! if @reconnect_with_consistent_host
+        end
       end
     end
 
-    attr_reader :global_retry_config
+    # Both attributes will have defined setters
+    attr_reader :retry_config, :reconnect_with_consistent_host
+    attr_accessor :connection
+
+    def retry_config=(retry_options)
+      @retry_config = default_retry_config.dup.merge!(retry_options)
+    end
+
+    def reconnect_with_consistent_host=(reconnect)
+      if (@reconnect_with_consistent_host = reconnect)
+        @initial_hostname = hostname
+        @initial_server_id = server_id
+      end
+    end
 
     private
 
+    def hostname
+      mysql_single_value(MYSQL_VAR_NAMES[:hostname])
+    end
+
+    def server_id
+      mysql_single_value(MYSQL_VAR_NAMES[:server_id])
+    end
+
+    def cloudsql?
+      mysql_single_value(MYSQL_VAR_NAMES[:version_comment]).include?(CLOUDSQL_VERSION_COMMENT)
+    end
+
+    def mysql_single_value(name)
+      query = Lhm::ProxySQLHelper.tagged("SELECT #{name} LIMIT 1")
+
+      @connection.execute(query).to_a.first.tap do |record|
+        return record&.first
+      end
+    end
+
+    def same_host_as_initial?
+      return @initial_server_id == server_id if cloudsql?
+      @initial_hostname == hostname
+    end
+
     def log_with_prefix(message, level = :info)
       message.prepend("[#{@log_prefix}] ") if @log_prefix
-      Lhm.logger.send(level, message)
+      Lhm.logger.public_send(level, message)
+    end
+
+    def reconnect_with_host_check!
+      log_with_prefix("Lost connection to MySQL, will retry to connect to same host")
+      begin
+        Retriable.retriable(host_retry_config) do
+          # tries to reconnect. On failure will trigger a retry
+          @connection.reconnect!
+
+          if same_host_as_initial?
+            # This is not an actual error, but controlled way to get the parent `Retriable.retriable` to retry
+            # the statement that failed (since the Retriable gem only retries on errors).
+            raise ReconnectToHostSuccessful.new("LHM successfully reconnected to initial host: #{@initial_hostname} (server_id: #{@initial_server_id})")
+          else
+            # New Master --> abort LHM (reconnecting will not change anything)
+            raise Lhm::Error.new("Reconnected to wrong host. Started migration on: #{@initial_hostname} (server_id: #{@initial_server_id}), but reconnected to: #{hostname} (server_id: #{@initial_server_id}).")
+          end
+        end
+      rescue StandardError => e
+        # The parent Retriable.retriable is configured to retry if it encounters an error with the success message.
+        # Therefore, if the connection is re-established successfully AND the host is the same, LHM can retry the query
+        # that originally failed.
+        raise e if reconnect_successful?(e)
+        # If the connection was not successful, the parent retriable will raise any error that originated from the
+        # `@connection.reconnect!`
+        # Therefore, this error will cause the LHM to abort
+        raise Lhm::Error.new("LHM tried the reconnection procedure but failed. Latest error: #{e.message}")
+      end
+    end
+
+    def reconnect_successful?(e)
+      e.is_a?(ReconnectToHostSuccessful)
     end
 
     # For a full list of configuration options see https://github.com/kamui/retriable
@@ -49,6 +147,11 @@ module Lhm
             /Lost connection to MySQL server during query/,
             /Max connect timeout reached/,
             /Unknown MySQL server host/,
+            /connection is locked to hostgroup/,
+            /The MySQL server is running with the --read-only option so it cannot execute this statement/,
+          ],
+          ReconnectToHostSuccessful => [
+            /#{RECONNECT_SUCCESSFUL_MESSAGE}/
           ]
         },
         multiplier: 1, # each successive interval grows by this factor
@@ -57,8 +160,29 @@ module Lhm
         rand_factor: 0, # percentage to randomize the next retry interval time
         max_elapsed_time: Float::INFINITY, # max total time in seconds that code is allowed to keep being retried
         on_retry: Proc.new do |exception, try_number, total_elapsed_time, next_interval|
-          log = "#{exception.class}: '#{exception.message}' - #{try_number} tries in #{total_elapsed_time} seconds and #{next_interval} seconds until the next try."
-          log_with_prefix(log, :info)
+          if reconnect_successful?(exception)
+            log_with_prefix("#{exception.message} -- triggering retry", :info)
+          else
+            log_with_prefix("#{exception.class}: '#{exception.message}' - #{try_number} tries in #{total_elapsed_time} seconds and #{next_interval} seconds until the next try.", :error)
+          end
+        end
+      }.freeze
+    end
+
+    def host_retry_config
+      {
+        on: {
+          StandardError => [
+            /Lost connection to MySQL server at 'reading initial communication packet'/
+          ]
+        },
+        multiplier: 1, # each successive interval grows by this factor
+        base_interval: 0.25, # the initial interval in seconds between tries.
+        tries: 20, # Number of attempts to make at running your code block (includes initial attempt).
+        rand_factor: 0, # percentage to randomize the next retry interval time
+        max_elapsed_time: Float::INFINITY, # max total time in seconds that code is allowed to keep being retried
+        on_retry: Proc.new do |exception, try_number, total_elapsed_time, next_interval|
+          log_with_prefix("#{exception.class}: '#{exception.message}' - #{try_number} tries in #{total_elapsed_time} seconds and #{next_interval} seconds until the next try.", :error)
         end
       }.freeze
     end

data/lib/lhm/throttler/slave_lag.rb

@@ -124,8 +124,8 @@ module Lhm
           else
             raise ArgumentError, "Expected #{config_proc.inspect} to respond to `call`"
           end
-        else # otherwise default to ActiveRecord provided config
-          ActiveRecord::Base.connection_pool.spec.config.dup
+        else
+          db_config
         end
         config.deep_symbolize_keys!
         config[:host] = @host
@@ -140,6 +140,23 @@ module Lhm
           [nil]
         end
       end
+
+      private
+
+      def db_config
+        if ar_supports_db_config?
+          ActiveRecord::Base.connection_pool.db_config.configuration_hash.dup
+        else
+          ActiveRecord::Base.connection_pool.spec.config.dup
+        end
+      end
+
+      def ar_supports_db_config?
+        # https://api.rubyonrails.org/v6.0/classes/ActiveRecord/ConnectionAdapters/ConnectionPool.html <-- has spec
+        # vs
+        # https://api.rubyonrails.org/v6.1/classes/ActiveRecord/ConnectionAdapters/ConnectionPool.html <-- has db_config
+        ActiveRecord::VERSION::MAJOR > 6 || ActiveRecord::VERSION::MAJOR == 6 && ActiveRecord::VERSION::MINOR >= 1
+      end
     end
   end
 end

data/lib/lhm/version.rb

@@ -2,5 +2,5 @@
 # Schmidt
 
 module Lhm
-  VERSION = '3.5.0'
+  VERSION = '3.5.4'
 end

data/lib/lhm.rb

@@ -8,6 +8,7 @@ require 'lhm/throttler'
 require 'lhm/version'
 require 'lhm/cleanup/current'
 require 'lhm/sql_retry'
+require 'lhm/proxysql_helper'
 require 'lhm/connection'
 require 'lhm/test_support'
 require 'lhm/railtie' if defined?(Rails::Railtie)
@@ -27,7 +28,7 @@ module Lhm
   extend Throttler
   extend self
 
-  DEFAULT_LOGGER_OPTIONS =  { level: Logger::INFO, file: STDOUT }
+  DEFAULT_LOGGER_OPTIONS = { level: Logger::INFO, file: STDOUT }
 
   # Alters a table with the changes described in the block
   #
@@ -45,15 +46,19 @@ module Lhm
   #   Use atomic switch to rename tables (defaults to: true)
   #   If using a version of mysql affected by atomic switch bug, LHM forces user
   #   to set this option (see SqlHelper#supports_atomic_switch?)
+  # @option options [Boolean] :reconnect_with_consistent_host
+  #   Active / Deactivate ProxySQL-aware reconnection procedure (default to: false)
   # @yield [Migrator] Yielded Migrator object records the changes
   # @return [Boolean] Returns true if the migration finishes
   # @raise [Error] Raises Lhm::Error in case of a error and aborts the migration
   def change_table(table_name, options = {}, &block)
-    origin = Table.parse(table_name, connection)
-    invoker = Invoker.new(origin, connection)
-    block.call(invoker.migrator)
-    invoker.run(options)
-    true
+    with_flags(options) do
+      origin = Table.parse(table_name, connection)
+      invoker = Invoker.new(origin, connection)
+      block.call(invoker.migrator)
+      invoker.run(options)
+      true
+    end
   end
 
   # Cleanup tables and triggers
@@ -82,16 +87,19 @@ module Lhm
     Lhm::Cleanup::Current.new(run, table_name, connection, options).execute
   end
 
+  # Setups DB connection
+  #
+  # @param [ActiveRecord::Base] connection ActiveRecord Connection
   def setup(connection)
-    @@connection = Lhm::Connection.new(connection: connection)
+    @@connection = Connection.new(connection: connection)
   end
 
+  # Returns DB connection (or initializes it if not created yet)
   def connection
-    @@connection ||=
-      begin
-        raise 'Please call Lhm.setup' unless defined?(ActiveRecord)
-        ActiveRecord::Base.connection
-      end
+    @@connection ||= begin
+                       raise 'Please call Lhm.setup' unless defined?(ActiveRecord)
+                       @@connection = Connection.new(connection: ActiveRecord::Base.connection)
+                     end
   end
 
   def self.logger=(new_logger)
@@ -133,4 +141,16 @@ module Lhm
       false
     end
   end
+
+  def with_flags(options)
+    old_flags = {
+      reconnect_with_consistent_host: Lhm.connection.reconnect_with_consistent_host,
+    }
+
+    Lhm.connection.reconnect_with_consistent_host = options[:reconnect_with_consistent_host] || false
+
+    yield
+  ensure
+    Lhm.connection.reconnect_with_consistent_host = old_flags[:reconnect_with_consistent_host]
+  end
 end

data/scripts/mysql/writer/create_users.sql

@@ -1,3 +1,6 @@
 # Creates replication user in Writer
+CREATE USER IF NOT EXISTS 'writer'@'%' IDENTIFIED BY 'password';
+CREATE USER IF NOT EXISTS 'reader'@'%' IDENTIFIED BY 'password';
+
 CREATE USER IF NOT EXISTS 'replication'@'%' IDENTIFIED BY 'password';
 GRANT REPLICATION SLAVE ON *.* TO' replication'@'%' IDENTIFIED BY 'password';