lhm-shopify 3.5.0 → 3.5.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (59) hide show
  1. checksums.yaml +4 -4
  2. data/.github/workflows/test.yml +17 -4
  3. data/.gitignore +0 -2
  4. data/Appraisals +24 -0
  5. data/Gemfile.lock +66 -0
  6. data/README.md +42 -0
  7. data/Rakefile +1 -0
  8. data/dev.yml +18 -3
  9. data/docker-compose.yml +15 -3
  10. data/gemfiles/activerecord_5.2.gemfile +9 -0
  11. data/gemfiles/activerecord_5.2.gemfile.lock +65 -0
  12. data/gemfiles/activerecord_6.0.gemfile +7 -0
  13. data/gemfiles/activerecord_6.0.gemfile.lock +67 -0
  14. data/gemfiles/activerecord_6.1.gemfile +7 -0
  15. data/gemfiles/activerecord_6.1.gemfile.lock +66 -0
  16. data/gemfiles/activerecord_7.0.0.alpha2.gemfile +7 -0
  17. data/gemfiles/activerecord_7.0.0.alpha2.gemfile.lock +64 -0
  18. data/lhm.gemspec +7 -3
  19. data/lib/lhm/atomic_switcher.rb +1 -1
  20. data/lib/lhm/chunk_insert.rb +2 -1
  21. data/lib/lhm/chunker.rb +3 -3
  22. data/lib/lhm/cleanup/current.rb +1 -1
  23. data/lib/lhm/connection.rb +50 -11
  24. data/lib/lhm/entangler.rb +2 -2
  25. data/lib/lhm/invoker.rb +2 -2
  26. data/lib/lhm/proxysql_helper.rb +10 -0
  27. data/lib/lhm/sql_retry.rb +126 -8
  28. data/lib/lhm/throttler/slave_lag.rb +19 -2
  29. data/lib/lhm/version.rb +1 -1
  30. data/lib/lhm.rb +22 -8
  31. data/scripts/mysql/writer/create_users.sql +3 -0
  32. data/spec/integration/atomic_switcher_spec.rb +27 -10
  33. data/spec/integration/chunk_insert_spec.rb +2 -1
  34. data/spec/integration/chunker_spec.rb +1 -1
  35. data/spec/integration/database.yml +10 -0
  36. data/spec/integration/entangler_spec.rb +3 -1
  37. data/spec/integration/integration_helper.rb +23 -5
  38. data/spec/integration/lhm_spec.rb +75 -0
  39. data/spec/integration/proxysql_spec.rb +34 -0
  40. data/spec/integration/sql_retry/db_connection_helper.rb +52 -0
  41. data/spec/integration/sql_retry/lock_wait_spec.rb +8 -6
  42. data/spec/integration/sql_retry/lock_wait_timeout_test_helper.rb +19 -9
  43. data/spec/integration/sql_retry/proxysql_helper.rb +22 -0
  44. data/spec/integration/sql_retry/retry_with_proxysql_spec.rb +108 -0
  45. data/spec/integration/toxiproxy_helper.rb +40 -0
  46. data/spec/test_helper.rb +21 -0
  47. data/spec/unit/chunk_insert_spec.rb +7 -2
  48. data/spec/unit/chunker_spec.rb +45 -42
  49. data/spec/unit/connection_spec.rb +22 -4
  50. data/spec/unit/entangler_spec.rb +41 -11
  51. data/spec/unit/throttler/slave_lag_spec.rb +13 -8
  52. metadata +76 -11
  53. data/gemfiles/ar-2.3_mysql.gemfile +0 -6
  54. data/gemfiles/ar-3.2_mysql.gemfile +0 -5
  55. data/gemfiles/ar-3.2_mysql2.gemfile +0 -5
  56. data/gemfiles/ar-4.0_mysql2.gemfile +0 -5
  57. data/gemfiles/ar-4.1_mysql2.gemfile +0 -5
  58. data/gemfiles/ar-4.2_mysql2.gemfile +0 -5
  59. data/gemfiles/ar-5.0_mysql2.gemfile +0 -5
@@ -54,7 +54,7 @@ module Lhm
54
54
 
55
55
  def execute_ddls
56
56
  ddls.each do |ddl|
57
- @connection.execute(ddl, @retry_config)
57
+ @connection.execute(ddl, should_retry: true, retry_options: @retry_config)
58
58
  end
59
59
  Lhm.logger.info("Dropped triggers on #{@lhm_triggers_for_origin.join(', ')}")
60
60
  Lhm.logger.info("Dropped tables #{@lhm_triggers_for_origin.join(', ')}")
@@ -1,4 +1,5 @@
1
1
  require 'delegate'
2
+ require 'lhm/sql_retry'
2
3
 
3
4
  module Lhm
4
5
  class Connection < SimpleDelegator
@@ -8,36 +9,74 @@ module Lhm
8
9
  alias connection __getobj__
9
10
  alias connection= __setobj__
10
11
 
11
- def initialize(connection:, default_log_prefix: nil, retry_options: {})
12
+ def initialize(connection:, default_log_prefix: nil, options: {}, retry_config: {})
12
13
  @default_log_prefix = default_log_prefix
13
- @retry_options = retry_options || default_retry_config
14
+ @retry_options = retry_config || default_retry_config
14
15
  @sql_retry = Lhm::SqlRetry.new(
15
16
  connection,
16
- retry_options,
17
+ options: retry_config,
18
+ reconnect_with_consistent_host: options[:reconnect_with_consistent_host] || false
17
19
  )
18
20
 
19
21
  # Creates delegation for the ActiveRecord Connection
20
22
  super(connection)
21
23
  end
22
24
 
23
- def execute(query, retry_options = {})
24
- exec_with_retries(:execute, query, retry_options)
25
+ def options=(options)
26
+ # If any other flags are added. Add the "processing" here
27
+ @sql_retry.reconnect_with_consistent_host = options[:reconnect_with_consistent_host] || false
25
28
  end
26
29
 
27
- def update(query, retry_options = {})
28
- exec_with_retries(:update, query, retry_options)
30
+ def execute(query, should_retry: false, retry_options: {})
31
+ if should_retry
32
+ exec_with_retries(:execute, query, retry_options)
33
+ else
34
+ exec(:execute, query)
35
+ end
29
36
  end
30
37
 
31
- def select_value(query, retry_options = {})
32
- exec_with_retries(:select_value, query, retry_options)
38
+ def update(query, should_retry: false, retry_options: {})
39
+ if should_retry
40
+ exec_with_retries(:update, query, retry_options)
41
+ else
42
+ exec(:update, query)
43
+ end
44
+ end
45
+
46
+ def select_value(query, should_retry: false, retry_options: {})
47
+ if should_retry
48
+ exec_with_retries(:select_value, query, retry_options)
49
+ else
50
+ exec(:select_value, query)
51
+ end
52
+ end
53
+
54
+ def select_values(query, should_retry: false, retry_options: {})
55
+ if should_retry
56
+ exec_with_retries(:select_values, query, retry_options)
57
+ else
58
+ exec(:select_values, query)
59
+ end
60
+ end
61
+
62
+ def select_one(query, should_retry: false, retry_options: {})
63
+ if should_retry
64
+ exec_with_retries(:select_one, query, retry_options)
65
+ else
66
+ exec(:select_one, query)
67
+ end
33
68
  end
34
69
 
35
70
  private
36
71
 
72
+ def exec(method, sql)
73
+ connection.public_send(method, Lhm::ProxySQLHelper.tagged(sql))
74
+ end
75
+
37
76
  def exec_with_retries(method, sql, retry_options = {})
38
77
  retry_options[:log_prefix] ||= file
39
78
  @sql_retry.with_retries(retry_options) do |conn|
40
- conn.public_send(method, sql)
79
+ conn.public_send(method, Lhm::ProxySQLHelper.tagged(sql))
41
80
  end
42
81
  end
43
82
 
@@ -51,7 +90,7 @@ module Lhm
51
90
 
52
91
  def relevant_caller
53
92
  lhm_stack = caller.select { |x| x.include?("/lhm") }
54
- first_candidate_index = lhm_stack.find_index {|line| !line.include?(__FILE__)}
93
+ first_candidate_index = lhm_stack.find_index { |line| !line.include?(__FILE__) }
55
94
 
56
95
  # Find the file that called the `#execute` (fallbacks to current file)
57
96
  return lhm_stack.first unless first_candidate_index
data/lib/lhm/entangler.rb CHANGED
@@ -86,14 +86,14 @@ module Lhm
86
86
 
87
87
  def before
88
88
  entangle.each do |stmt|
89
- @connection.execute(stmt, @retry_options)
89
+ @connection.execute(stmt, should_retry: true, retry_options: @retry_options)
90
90
  end
91
91
  Lhm.logger.info("Created triggers on #{@origin.name}")
92
92
  end
93
93
 
94
94
  def after
95
95
  untangle.each do |stmt|
96
- @connection.execute(stmt, @retry_options)
96
+ @connection.execute(stmt, should_retry: true, retry_options: @retry_options)
97
97
  end
98
98
  Lhm.logger.info("Dropped triggers on #{@origin.name}")
99
99
  end
data/lib/lhm/invoker.rb CHANGED
@@ -16,8 +16,8 @@ module Lhm
16
16
  class Invoker
17
17
  include SqlHelper
18
18
  LOCK_WAIT_TIMEOUT_DELTA = 10
19
- INNODB_LOCK_WAIT_TIMEOUT_MAX=1073741824.freeze # https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_lock_wait_timeout
20
- LOCK_WAIT_TIMEOUT_MAX=31536000.freeze # https://dev.mysql.com/doc/refman/5.7/en/server-system-variables.html
19
+ INNODB_LOCK_WAIT_TIMEOUT_MAX=1073741824.freeze # https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_lock_wait_timeout
20
+ LOCK_WAIT_TIMEOUT_MAX=31536000.freeze # https://dev.mysql.com/doc/refman/5.7/en/server-system-variables.html
21
21
 
22
22
  attr_reader :migrator, :connection
23
23
 
@@ -0,0 +1,10 @@
1
+ module Lhm
2
+ module ProxySQLHelper
3
+ extend self
4
+ ANNOTATION = "/*maintenance:lhm*/"
5
+
6
+ def tagged(sql)
7
+ "#{ANNOTATION}#{sql}"
8
+ end
9
+ end
10
+ end
data/lib/lhm/sql_retry.rb CHANGED
@@ -1,5 +1,6 @@
1
1
  require 'retriable'
2
2
  require 'lhm/sql_helper'
3
+ require 'lhm/proxysql_helper'
3
4
 
4
5
  module Lhm
5
6
  # SqlRetry standardizes the interface for retry behavior in components like
@@ -15,26 +16,117 @@ module Lhm
15
16
  # https://github.com/kamui/retriable. Additionally, a "log_prefix" option,
16
17
  # which is unique to SqlRetry can be used to prefix log output.
17
18
  class SqlRetry
18
- def initialize(connection, options = {})
19
+ RECONNECT_SUCCESSFUL_MESSAGE = "LHM successfully reconnected to initial host:"
20
+ CLOUDSQL_VERSION_COMMENT = "(Google)"
21
+
22
+ MYSQL_VAR_NAMES = {
23
+ hostname: "@@global.hostname",
24
+ server_id: "@@global.server_id",
25
+ version_comment: "@@version_comment",
26
+ }
27
+
28
+ # This internal error is used to trigger retries from the parent Retriable.retriable in #with_retries
29
+ class ReconnectToHostSuccessful < Lhm::Error; end
30
+
31
+ def initialize(connection, options: {}, reconnect_with_consistent_host: true)
19
32
  @connection = connection
33
+ @log_prefix = options.delete(:log_prefix)
20
34
  @global_retry_config = default_retry_config.dup.merge!(options)
35
+ if (@reconnect_with_consistent_host = reconnect_with_consistent_host)
36
+ @initial_hostname = hostname
37
+ @initial_server_id = server_id
38
+ end
21
39
  end
22
40
 
41
+ # Complete explanation of algorithm: https://github.com/Shopify/lhm/pull/112
23
42
  def with_retries(retry_config = {})
24
- cnf = @global_retry_config.dup.merge!(retry_config)
25
- @log_prefix = cnf.delete(:log_prefix) || "SQL Retry"
26
- Retriable.retriable(cnf) do
27
- yield(@connection)
43
+ @log_prefix = retry_config.delete(:log_prefix)
44
+
45
+ retry_config = @global_retry_config.dup.merge!(retry_config)
46
+
47
+ Retriable.retriable(retry_config) do
48
+ # Using begin -> rescue -> end for Ruby 2.4 compatibility
49
+ begin
50
+ if @reconnect_with_consistent_host
51
+ raise Lhm::Error.new("Could not reconnected to initial MySQL host. Aborting to avoid data-loss") unless same_host_as_initial?
52
+ end
53
+
54
+ yield(@connection)
55
+ rescue StandardError => e
56
+ # Not all errors should trigger a reconnect. Some errors such be raised and abort the LHM (such as reconnecting to the wrong host).
57
+ # The error will be raised the connection is still active (i.e. no need to reconnect) or if the connection is
58
+ # dead (i.e. not active) and @reconnect_with_host is false (i.e. instructed not to reconnect)
59
+ raise e if @connection.active? || (!@connection.active? && !@reconnect_with_consistent_host)
60
+ reconnect_with_host_check! if @reconnect_with_consistent_host
61
+ end
28
62
  end
29
63
  end
30
64
 
31
65
  attr_reader :global_retry_config
66
+ attr_accessor :reconnect_with_consistent_host
32
67
 
33
68
  private
34
69
 
70
+ def hostname
71
+ mysql_single_value(MYSQL_VAR_NAMES[:hostname])
72
+ end
73
+
74
+ def server_id
75
+ mysql_single_value(MYSQL_VAR_NAMES[:server_id])
76
+ end
77
+
78
+ def cloudsql?
79
+ mysql_single_value(MYSQL_VAR_NAMES[:version_comment]).include?(CLOUDSQL_VERSION_COMMENT)
80
+ end
81
+
82
+ def mysql_single_value(name)
83
+ query = Lhm::ProxySQLHelper.tagged("SELECT #{name} LIMIT 1")
84
+
85
+ @connection.execute(query).to_a.first.tap do |record|
86
+ return record&.first
87
+ end
88
+ end
89
+
90
+ def same_host_as_initial?
91
+ return @initial_server_id == server_id if cloudsql?
92
+ @initial_hostname == hostname
93
+ end
94
+
35
95
  def log_with_prefix(message, level = :info)
36
96
  message.prepend("[#{@log_prefix}] ") if @log_prefix
37
- Lhm.logger.send(level, message)
97
+ Lhm.logger.public_send(level, message)
98
+ end
99
+
100
+ def reconnect_with_host_check!
101
+ log_with_prefix("Lost connection to MySQL, will retry to connect to same host")
102
+ begin
103
+ Retriable.retriable(host_retry_config) do
104
+ # tries to reconnect. On failure will trigger a retry
105
+ @connection.reconnect!
106
+
107
+ if same_host_as_initial?
108
+ # This is not an actual error, but controlled way to get the parent `Retriable.retriable` to retry
109
+ # the statement that failed (since the Retriable gem only retries on errors).
110
+ raise ReconnectToHostSuccessful.new("LHM successfully reconnected to initial host: #{@initial_hostname} (server_id: #{@initial_server_id})")
111
+ else
112
+ # New Master --> abort LHM (reconnecting will not change anything)
113
+ raise Lhm::Error.new("Reconnected to wrong host. Started migration on: #{@initial_hostname} (server_id: #{@initial_server_id}), but reconnected to: #{hostname} (server_id: #{@initial_server_id}).")
114
+ end
115
+ end
116
+ rescue StandardError => e
117
+ # The parent Retriable.retriable is configured to retry if it encounters an error with the success message.
118
+ # Therefore, if the connection is re-established successfully AND the host is the same, LHM can retry the query
119
+ # that originally failed.
120
+ raise e if reconnect_successful?(e)
121
+ # If the connection was not successful, the parent retriable will raise any error that originated from the
122
+ # `@connection.reconnect!`
123
+ # Therefore, this error will cause the LHM to abort
124
+ raise Lhm::Error.new("LHM tried the reconnection procedure but failed. Latest error: #{e.message}")
125
+ end
126
+ end
127
+
128
+ def reconnect_successful?(e)
129
+ e.is_a?(ReconnectToHostSuccessful)
38
130
  end
39
131
 
40
132
  # For a full list of configuration options see https://github.com/kamui/retriable
@@ -49,6 +141,11 @@ module Lhm
49
141
  /Lost connection to MySQL server during query/,
50
142
  /Max connect timeout reached/,
51
143
  /Unknown MySQL server host/,
144
+ /connection is locked to hostgroup/,
145
+ /The MySQL server is running with the --read-only option so it cannot execute this statement/,
146
+ ],
147
+ ReconnectToHostSuccessful => [
148
+ /#{RECONNECT_SUCCESSFUL_MESSAGE}/
52
149
  ]
53
150
  },
54
151
  multiplier: 1, # each successive interval grows by this factor
@@ -57,8 +154,29 @@ module Lhm
57
154
  rand_factor: 0, # percentage to randomize the next retry interval time
58
155
  max_elapsed_time: Float::INFINITY, # max total time in seconds that code is allowed to keep being retried
59
156
  on_retry: Proc.new do |exception, try_number, total_elapsed_time, next_interval|
60
- log = "#{exception.class}: '#{exception.message}' - #{try_number} tries in #{total_elapsed_time} seconds and #{next_interval} seconds until the next try."
61
- log_with_prefix(log, :info)
157
+ if reconnect_successful?(exception)
158
+ log_with_prefix("#{exception.message} -- triggering retry", :info)
159
+ else
160
+ log_with_prefix("#{exception.class}: '#{exception.message}' - #{try_number} tries in #{total_elapsed_time} seconds and #{next_interval} seconds until the next try.", :error)
161
+ end
162
+ end
163
+ }.freeze
164
+ end
165
+
166
+ def host_retry_config
167
+ {
168
+ on: {
169
+ StandardError => [
170
+ /Lost connection to MySQL server at 'reading initial communication packet'/
171
+ ]
172
+ },
173
+ multiplier: 1, # each successive interval grows by this factor
174
+ base_interval: 0.25, # the initial interval in seconds between tries.
175
+ tries: 20, # Number of attempts to make at running your code block (includes initial attempt).
176
+ rand_factor: 0, # percentage to randomize the next retry interval time
177
+ max_elapsed_time: Float::INFINITY, # max total time in seconds that code is allowed to keep being retried
178
+ on_retry: Proc.new do |exception, try_number, total_elapsed_time, next_interval|
179
+ log_with_prefix("#{exception.class}: '#{exception.message}' - #{try_number} tries in #{total_elapsed_time} seconds and #{next_interval} seconds until the next try.", :error)
62
180
  end
63
181
  }.freeze
64
182
  end
@@ -124,8 +124,8 @@ module Lhm
124
124
  else
125
125
  raise ArgumentError, "Expected #{config_proc.inspect} to respond to `call`"
126
126
  end
127
- else # otherwise default to ActiveRecord provided config
128
- ActiveRecord::Base.connection_pool.spec.config.dup
127
+ else
128
+ db_config
129
129
  end
130
130
  config.deep_symbolize_keys!
131
131
  config[:host] = @host
@@ -140,6 +140,23 @@ module Lhm
140
140
  [nil]
141
141
  end
142
142
  end
143
+
144
+ private
145
+
146
+ def db_config
147
+ if ar_supports_db_config?
148
+ ActiveRecord::Base.connection_pool.db_config.configuration_hash.dup
149
+ else
150
+ ActiveRecord::Base.connection_pool.spec.config.dup
151
+ end
152
+ end
153
+
154
+ def ar_supports_db_config?
155
+ # https://api.rubyonrails.org/v6.0/classes/ActiveRecord/ConnectionAdapters/ConnectionPool.html <-- has spec
156
+ # vs
157
+ # https://api.rubyonrails.org/v6.1/classes/ActiveRecord/ConnectionAdapters/ConnectionPool.html <-- has db_config
158
+ ActiveRecord::VERSION::MAJOR > 6 || ActiveRecord::VERSION::MAJOR == 6 && ActiveRecord::VERSION::MINOR >= 1
159
+ end
143
160
  end
144
161
  end
145
162
  end
data/lib/lhm/version.rb CHANGED
@@ -2,5 +2,5 @@
2
2
  # Schmidt
3
3
 
4
4
  module Lhm
5
- VERSION = '3.5.0'
5
+ VERSION = '3.5.1'
6
6
  end
data/lib/lhm.rb CHANGED
@@ -8,6 +8,7 @@ require 'lhm/throttler'
8
8
  require 'lhm/version'
9
9
  require 'lhm/cleanup/current'
10
10
  require 'lhm/sql_retry'
11
+ require 'lhm/proxysql_helper'
11
12
  require 'lhm/connection'
12
13
  require 'lhm/test_support'
13
14
  require 'lhm/railtie' if defined?(Rails::Railtie)
@@ -82,16 +83,29 @@ module Lhm
82
83
  Lhm::Cleanup::Current.new(run, table_name, connection, options).execute
83
84
  end
84
85
 
85
- def setup(connection)
86
- @@connection = Lhm::Connection.new(connection: connection)
86
+ # Setups DB connection
87
+ #
88
+ # @param [ActiveRecord::Base] connection ActiveRecord Connection
89
+ # @param [Hash] connection_options Optional options (defaults to: empty hash)
90
+ # @option connection_options [Boolean] :reconnect_with_consistent_host
91
+ # Active / Deactivate ProxySQL-aware reconnection procedure (default to: false)
92
+ def setup(connection, connection_options = {})
93
+ @@connection = Connection.new(connection: connection, options: connection_options)
87
94
  end
88
95
 
89
- def connection
90
- @@connection ||=
91
- begin
92
- raise 'Please call Lhm.setup' unless defined?(ActiveRecord)
93
- ActiveRecord::Base.connection
94
- end
96
+ # Setups DB connection
97
+ #
98
+ # @param [Hash] connection_options Optional options (defaults to: empty hash)
99
+ # @option connection_options [Boolean] :reconnect_with_consistent_host
100
+ # Active / Deactivate ProxySQL-aware reconnection procedure (default to: false)
101
+ def connection(connection_options = nil)
102
+ if @@connection.nil?
103
+ raise 'Please call Lhm.setup' unless defined?(ActiveRecord)
104
+ @@connection = Connection.new(connection: ActiveRecord::Base.connection, options: connection_options || {})
105
+ else
106
+ @@connection.options = connection_options unless connection_options.nil?
107
+ end
108
+ @@connection
95
109
  end
96
110
 
97
111
  def self.logger=(new_logger)
@@ -1,3 +1,6 @@
1
1
  # Creates replication user in Writer
2
+ CREATE USER IF NOT EXISTS 'writer'@'%' IDENTIFIED BY 'password';
3
+ CREATE USER IF NOT EXISTS 'reader'@'%' IDENTIFIED BY 'password';
4
+
2
5
  CREATE USER IF NOT EXISTS 'replication'@'%' IDENTIFIED BY 'password';
3
6
  GRANT REPLICATION SLAVE ON *.* TO' replication'@'%' IDENTIFIED BY 'password';
@@ -16,9 +16,9 @@ describe Lhm::AtomicSwitcher do
16
16
  describe 'switching' do
17
17
  before(:each) do
18
18
  Thread.abort_on_exception = true
19
- @origin = table_create('origin')
19
+ @origin = table_create('origin')
20
20
  @destination = table_create('destination')
21
- @migration = Lhm::Migration.new(@origin, @destination)
21
+ @migration = Lhm::Migration.new(@origin, @destination)
22
22
  @logs = StringIO.new
23
23
  Lhm.logger = Logger.new(@logs)
24
24
  @connection.execute('SET GLOBAL innodb_lock_wait_timeout=3')
@@ -32,11 +32,17 @@ describe Lhm::AtomicSwitcher do
32
32
  it 'should retry and log on lock wait timeouts' do
33
33
  ar_connection = mock()
34
34
  ar_connection.stubs(:data_source_exists?).returns(true)
35
- ar_connection.stubs(:execute).raises(ActiveRecord::StatementInvalid, 'Lock wait timeout exceeded; try restarting transaction.').then.returns(true)
35
+ ar_connection.stubs(:active?).returns(true)
36
+ ar_connection.stubs(:execute).returns([["dummy"]], [["dummy"]], [["dummy"]])
37
+ .then
38
+ .raises(ActiveRecord::StatementInvalid, 'Lock wait timeout exceeded; try restarting transaction.')
39
+ .then
40
+ .returns([["dummy"]]) # Matches initial host -> triggers retry
41
+
42
+ connection = Lhm::Connection.new(connection: ar_connection, options: {reconnect_with_consistent_host: true})
36
43
 
37
- connection = Lhm::Connection.new(connection: ar_connection)
38
44
 
39
- switcher = Lhm::AtomicSwitcher.new(@migration, connection, retriable: {base_interval: 0})
45
+ switcher = Lhm::AtomicSwitcher.new(@migration, connection, retriable: { tries: 3, base_interval: 0 })
40
46
 
41
47
  assert switcher.run
42
48
 
@@ -50,11 +56,22 @@ describe Lhm::AtomicSwitcher do
50
56
  it 'should give up on lock wait timeouts after a configured number of tries' do
51
57
  ar_connection = mock()
52
58
  ar_connection.stubs(:data_source_exists?).returns(true)
53
- ar_connection.stubs(:execute).twice.raises(ActiveRecord::StatementInvalid, 'Lock wait timeout exceeded; try restarting transaction.')
54
-
55
- connection = Lhm::Connection.new(connection: ar_connection)
56
-
57
- switcher = Lhm::AtomicSwitcher.new(@migration, connection, retriable: {tries: 2, base_interval: 0})
59
+ ar_connection.stubs(:active?).returns(true)
60
+ ar_connection.stubs(:execute).returns([["dummy"]], [["dummy"]], [["dummy"]])
61
+ .then
62
+ .raises(ActiveRecord::StatementInvalid, 'Lock wait timeout exceeded; try restarting transaction.')
63
+ .then
64
+ .returns([["dummy"]]) # triggers retry 1
65
+ .then
66
+ .raises(ActiveRecord::StatementInvalid, 'Lock wait timeout exceeded; try restarting transaction.')
67
+ .then
68
+ .returns([["dummy"]]) # triggers retry 2
69
+ .then
70
+ .raises(ActiveRecord::StatementInvalid, 'Lock wait timeout exceeded; try restarting transaction.') # triggers retry 2
71
+
72
+ connection = Lhm::Connection.new(connection: ar_connection, options: {reconnect_with_consistent_host: true})
73
+
74
+ switcher = Lhm::AtomicSwitcher.new(@migration, connection, retriable: { tries: 2, base_interval: 0 })
58
75
 
59
76
  assert_raises(ActiveRecord::StatementInvalid) { switcher.run }
60
77
  end
@@ -11,7 +11,8 @@ describe Lhm::ChunkInsert do
11
11
  @destination = table_create(:destination)
12
12
  @migration = Lhm::Migration.new(@origin, @destination)
13
13
  execute("insert into origin set id = 1001")
14
- @instance = Lhm::ChunkInsert.new(@migration, connection, 1001, 1001)
14
+ @connection = Lhm::Connection.new(connection: connection)
15
+ @instance = Lhm::ChunkInsert.new(@migration, @connection, 1001, 1001)
15
16
  end
16
17
 
17
18
  it "returns the count" do
@@ -223,7 +223,7 @@ describe Lhm::Chunker do
223
223
 
224
224
  if master_slave_mode?
225
225
  def throttler.slave_connection(slave)
226
- config = ActiveRecord::Base.connection_pool.spec.config.dup
226
+ config = ActiveRecord::Base.connection_pool.db_config.configuration_hash.dup
227
227
  config[:host] = slave
228
228
  config[:port] = 3307
229
229
  ActiveRecord::Base.send('mysql2_connection', config)
@@ -13,3 +13,13 @@ proxysql:
13
13
  user: root
14
14
  password: password
15
15
  port: 33005
16
+ master_toxic:
17
+ host: toxiproxy
18
+ user: root
19
+ password: password
20
+ port: 22220
21
+ proxysql_toxic:
22
+ host: toxiproxy
23
+ user: root
24
+ password: password
25
+ port: 22222
@@ -6,6 +6,7 @@ require File.expand_path(File.dirname(__FILE__)) + '/integration_helper'
6
6
  require 'lhm/table'
7
7
  require 'lhm/migration'
8
8
  require 'lhm/entangler'
9
+ require 'lhm/connection'
9
10
 
10
11
  describe Lhm::Entangler do
11
12
  include IntegrationHelper
@@ -17,7 +18,8 @@ describe Lhm::Entangler do
17
18
  @origin = table_create('origin')
18
19
  @destination = table_create('destination')
19
20
  @migration = Lhm::Migration.new(@origin, @destination)
20
- @entangler = Lhm::Entangler.new(@migration, connection)
21
+ @connection = Lhm::Connection.new(connection: connection)
22
+ @entangler = Lhm::Entangler.new(@migration, @connection)
21
23
  end
22
24
 
23
25
  it 'should replay inserts from origin into destination' do
@@ -35,6 +35,15 @@ module IntegrationHelper
35
35
  @connection
36
36
  end
37
37
 
38
+ def connect_proxysql!
39
+ connect!(
40
+ '127.0.0.1',
41
+ $db_config['proxysql']['port'],
42
+ $db_config['proxysql']['user'],
43
+ $db_config['proxysql']['password'],
44
+ )
45
+ end
46
+
38
47
  def connect_master!
39
48
  connect!(
40
49
  '127.0.0.1',
@@ -53,14 +62,23 @@ module IntegrationHelper
53
62
  )
54
63
  end
55
64
 
56
- def connect!(hostname, port, user, password)
57
- adapter = Lhm::Connection.new(connection: ar_conn(hostname, port, user, password))
58
- Lhm.setup(adapter)
65
+ def connect_master_with_toxiproxy!(with_retry: false)
66
+ connect!(
67
+ '127.0.0.1',
68
+ $db_config['master_toxic']['port'],
69
+ $db_config['master_toxic']['user'],
70
+ $db_config['master_toxic']['password'],
71
+ with_retry)
72
+ end
73
+
74
+ def connect!(hostname, port, user, password, with_retry = false)
75
+ adapter = ar_conn(hostname, port, user, password)
76
+ Lhm.setup(adapter,{reconnect_with_consistent_host: with_retry})
59
77
  unless defined?(@@cleaned_up)
60
78
  Lhm.cleanup(true)
61
79
  @@cleaned_up = true
62
80
  end
63
- @connection = adapter
81
+ @connection = Lhm.connection
64
82
  end
65
83
 
66
84
  def ar_conn(host, port, user, password)
@@ -119,7 +137,7 @@ module IntegrationHelper
119
137
  # Helps testing behaviour when another client locks the db
120
138
  def start_locking_thread(lock_for, queue, locking_query)
121
139
  Thread.new do
122
- conn = Mysql2::Client.new(host: '127.0.0.1', database: $db_name, user: 'root', port: 3306)
140
+ conn = new_mysql_connection
123
141
  conn.query('BEGIN')
124
142
  conn.query(locking_query)
125
143
  queue.push(true)