lhm-shopify 3.5.0 → 3.5.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.github/workflows/test.yml +17 -4
- data/.gitignore +0 -2
- data/Appraisals +24 -0
- data/Gemfile.lock +66 -0
- data/README.md +42 -0
- data/Rakefile +1 -0
- data/dev.yml +18 -3
- data/docker-compose.yml +15 -3
- data/gemfiles/activerecord_5.2.gemfile +9 -0
- data/gemfiles/activerecord_5.2.gemfile.lock +65 -0
- data/gemfiles/activerecord_6.0.gemfile +7 -0
- data/gemfiles/activerecord_6.0.gemfile.lock +67 -0
- data/gemfiles/activerecord_6.1.gemfile +7 -0
- data/gemfiles/activerecord_6.1.gemfile.lock +66 -0
- data/gemfiles/activerecord_7.0.0.alpha2.gemfile +7 -0
- data/gemfiles/activerecord_7.0.0.alpha2.gemfile.lock +64 -0
- data/lhm.gemspec +7 -3
- data/lib/lhm/atomic_switcher.rb +1 -1
- data/lib/lhm/chunk_insert.rb +2 -1
- data/lib/lhm/chunker.rb +3 -3
- data/lib/lhm/cleanup/current.rb +1 -1
- data/lib/lhm/connection.rb +50 -11
- data/lib/lhm/entangler.rb +2 -2
- data/lib/lhm/invoker.rb +2 -2
- data/lib/lhm/proxysql_helper.rb +10 -0
- data/lib/lhm/sql_retry.rb +126 -8
- data/lib/lhm/throttler/slave_lag.rb +19 -2
- data/lib/lhm/version.rb +1 -1
- data/lib/lhm.rb +22 -8
- data/scripts/mysql/writer/create_users.sql +3 -0
- data/spec/integration/atomic_switcher_spec.rb +27 -10
- data/spec/integration/chunk_insert_spec.rb +2 -1
- data/spec/integration/chunker_spec.rb +1 -1
- data/spec/integration/database.yml +10 -0
- data/spec/integration/entangler_spec.rb +3 -1
- data/spec/integration/integration_helper.rb +23 -5
- data/spec/integration/lhm_spec.rb +75 -0
- data/spec/integration/proxysql_spec.rb +34 -0
- data/spec/integration/sql_retry/db_connection_helper.rb +52 -0
- data/spec/integration/sql_retry/lock_wait_spec.rb +8 -6
- data/spec/integration/sql_retry/lock_wait_timeout_test_helper.rb +19 -9
- data/spec/integration/sql_retry/proxysql_helper.rb +22 -0
- data/spec/integration/sql_retry/retry_with_proxysql_spec.rb +108 -0
- data/spec/integration/toxiproxy_helper.rb +40 -0
- data/spec/test_helper.rb +21 -0
- data/spec/unit/chunk_insert_spec.rb +7 -2
- data/spec/unit/chunker_spec.rb +45 -42
- data/spec/unit/connection_spec.rb +22 -4
- data/spec/unit/entangler_spec.rb +41 -11
- data/spec/unit/throttler/slave_lag_spec.rb +13 -8
- metadata +76 -11
- data/gemfiles/ar-2.3_mysql.gemfile +0 -6
- data/gemfiles/ar-3.2_mysql.gemfile +0 -5
- data/gemfiles/ar-3.2_mysql2.gemfile +0 -5
- data/gemfiles/ar-4.0_mysql2.gemfile +0 -5
- data/gemfiles/ar-4.1_mysql2.gemfile +0 -5
- data/gemfiles/ar-4.2_mysql2.gemfile +0 -5
- data/gemfiles/ar-5.0_mysql2.gemfile +0 -5
data/lib/lhm/cleanup/current.rb
CHANGED
@@ -54,7 +54,7 @@ module Lhm
|
|
54
54
|
|
55
55
|
def execute_ddls
|
56
56
|
ddls.each do |ddl|
|
57
|
-
@connection.execute(ddl, @retry_config)
|
57
|
+
@connection.execute(ddl, should_retry: true, retry_options: @retry_config)
|
58
58
|
end
|
59
59
|
Lhm.logger.info("Dropped triggers on #{@lhm_triggers_for_origin.join(', ')}")
|
60
60
|
Lhm.logger.info("Dropped tables #{@lhm_triggers_for_origin.join(', ')}")
|
data/lib/lhm/connection.rb
CHANGED
@@ -1,4 +1,5 @@
|
|
1
1
|
require 'delegate'
|
2
|
+
require 'lhm/sql_retry'
|
2
3
|
|
3
4
|
module Lhm
|
4
5
|
class Connection < SimpleDelegator
|
@@ -8,36 +9,74 @@ module Lhm
|
|
8
9
|
alias connection __getobj__
|
9
10
|
alias connection= __setobj__
|
10
11
|
|
11
|
-
def initialize(connection:, default_log_prefix: nil,
|
12
|
+
def initialize(connection:, default_log_prefix: nil, options: {}, retry_config: {})
|
12
13
|
@default_log_prefix = default_log_prefix
|
13
|
-
@retry_options =
|
14
|
+
@retry_options = retry_config || default_retry_config
|
14
15
|
@sql_retry = Lhm::SqlRetry.new(
|
15
16
|
connection,
|
16
|
-
|
17
|
+
options: retry_config,
|
18
|
+
reconnect_with_consistent_host: options[:reconnect_with_consistent_host] || false
|
17
19
|
)
|
18
20
|
|
19
21
|
# Creates delegation for the ActiveRecord Connection
|
20
22
|
super(connection)
|
21
23
|
end
|
22
24
|
|
23
|
-
def
|
24
|
-
|
25
|
+
def options=(options)
|
26
|
+
# If any other flags are added. Add the "processing" here
|
27
|
+
@sql_retry.reconnect_with_consistent_host = options[:reconnect_with_consistent_host] || false
|
25
28
|
end
|
26
29
|
|
27
|
-
def
|
28
|
-
|
30
|
+
def execute(query, should_retry: false, retry_options: {})
|
31
|
+
if should_retry
|
32
|
+
exec_with_retries(:execute, query, retry_options)
|
33
|
+
else
|
34
|
+
exec(:execute, query)
|
35
|
+
end
|
29
36
|
end
|
30
37
|
|
31
|
-
def
|
32
|
-
|
38
|
+
def update(query, should_retry: false, retry_options: {})
|
39
|
+
if should_retry
|
40
|
+
exec_with_retries(:update, query, retry_options)
|
41
|
+
else
|
42
|
+
exec(:update, query)
|
43
|
+
end
|
44
|
+
end
|
45
|
+
|
46
|
+
def select_value(query, should_retry: false, retry_options: {})
|
47
|
+
if should_retry
|
48
|
+
exec_with_retries(:select_value, query, retry_options)
|
49
|
+
else
|
50
|
+
exec(:select_value, query)
|
51
|
+
end
|
52
|
+
end
|
53
|
+
|
54
|
+
def select_values(query, should_retry: false, retry_options: {})
|
55
|
+
if should_retry
|
56
|
+
exec_with_retries(:select_values, query, retry_options)
|
57
|
+
else
|
58
|
+
exec(:select_values, query)
|
59
|
+
end
|
60
|
+
end
|
61
|
+
|
62
|
+
def select_one(query, should_retry: false, retry_options: {})
|
63
|
+
if should_retry
|
64
|
+
exec_with_retries(:select_one, query, retry_options)
|
65
|
+
else
|
66
|
+
exec(:select_one, query)
|
67
|
+
end
|
33
68
|
end
|
34
69
|
|
35
70
|
private
|
36
71
|
|
72
|
+
def exec(method, sql)
|
73
|
+
connection.public_send(method, Lhm::ProxySQLHelper.tagged(sql))
|
74
|
+
end
|
75
|
+
|
37
76
|
def exec_with_retries(method, sql, retry_options = {})
|
38
77
|
retry_options[:log_prefix] ||= file
|
39
78
|
@sql_retry.with_retries(retry_options) do |conn|
|
40
|
-
conn.public_send(method, sql)
|
79
|
+
conn.public_send(method, Lhm::ProxySQLHelper.tagged(sql))
|
41
80
|
end
|
42
81
|
end
|
43
82
|
|
@@ -51,7 +90,7 @@ module Lhm
|
|
51
90
|
|
52
91
|
def relevant_caller
|
53
92
|
lhm_stack = caller.select { |x| x.include?("/lhm") }
|
54
|
-
first_candidate_index = lhm_stack.find_index {|line| !line.include?(__FILE__)}
|
93
|
+
first_candidate_index = lhm_stack.find_index { |line| !line.include?(__FILE__) }
|
55
94
|
|
56
95
|
# Find the file that called the `#execute` (fallbacks to current file)
|
57
96
|
return lhm_stack.first unless first_candidate_index
|
data/lib/lhm/entangler.rb
CHANGED
@@ -86,14 +86,14 @@ module Lhm
|
|
86
86
|
|
87
87
|
def before
|
88
88
|
entangle.each do |stmt|
|
89
|
-
@connection.execute(stmt, @retry_options)
|
89
|
+
@connection.execute(stmt, should_retry: true, retry_options: @retry_options)
|
90
90
|
end
|
91
91
|
Lhm.logger.info("Created triggers on #{@origin.name}")
|
92
92
|
end
|
93
93
|
|
94
94
|
def after
|
95
95
|
untangle.each do |stmt|
|
96
|
-
@connection.execute(stmt, @retry_options)
|
96
|
+
@connection.execute(stmt, should_retry: true, retry_options: @retry_options)
|
97
97
|
end
|
98
98
|
Lhm.logger.info("Dropped triggers on #{@origin.name}")
|
99
99
|
end
|
data/lib/lhm/invoker.rb
CHANGED
@@ -16,8 +16,8 @@ module Lhm
|
|
16
16
|
class Invoker
|
17
17
|
include SqlHelper
|
18
18
|
LOCK_WAIT_TIMEOUT_DELTA = 10
|
19
|
-
|
20
|
-
|
19
|
+
INNODB_LOCK_WAIT_TIMEOUT_MAX=1073741824.freeze # https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_lock_wait_timeout
|
20
|
+
LOCK_WAIT_TIMEOUT_MAX=31536000.freeze # https://dev.mysql.com/doc/refman/5.7/en/server-system-variables.html
|
21
21
|
|
22
22
|
attr_reader :migrator, :connection
|
23
23
|
|
data/lib/lhm/sql_retry.rb
CHANGED
@@ -1,5 +1,6 @@
|
|
1
1
|
require 'retriable'
|
2
2
|
require 'lhm/sql_helper'
|
3
|
+
require 'lhm/proxysql_helper'
|
3
4
|
|
4
5
|
module Lhm
|
5
6
|
# SqlRetry standardizes the interface for retry behavior in components like
|
@@ -15,26 +16,117 @@ module Lhm
|
|
15
16
|
# https://github.com/kamui/retriable. Additionally, a "log_prefix" option,
|
16
17
|
# which is unique to SqlRetry can be used to prefix log output.
|
17
18
|
class SqlRetry
|
18
|
-
|
19
|
+
RECONNECT_SUCCESSFUL_MESSAGE = "LHM successfully reconnected to initial host:"
|
20
|
+
CLOUDSQL_VERSION_COMMENT = "(Google)"
|
21
|
+
|
22
|
+
MYSQL_VAR_NAMES = {
|
23
|
+
hostname: "@@global.hostname",
|
24
|
+
server_id: "@@global.server_id",
|
25
|
+
version_comment: "@@version_comment",
|
26
|
+
}
|
27
|
+
|
28
|
+
# This internal error is used to trigger retries from the parent Retriable.retriable in #with_retries
|
29
|
+
class ReconnectToHostSuccessful < Lhm::Error; end
|
30
|
+
|
31
|
+
def initialize(connection, options: {}, reconnect_with_consistent_host: true)
|
19
32
|
@connection = connection
|
33
|
+
@log_prefix = options.delete(:log_prefix)
|
20
34
|
@global_retry_config = default_retry_config.dup.merge!(options)
|
35
|
+
if (@reconnect_with_consistent_host = reconnect_with_consistent_host)
|
36
|
+
@initial_hostname = hostname
|
37
|
+
@initial_server_id = server_id
|
38
|
+
end
|
21
39
|
end
|
22
40
|
|
41
|
+
# Complete explanation of algorithm: https://github.com/Shopify/lhm/pull/112
|
23
42
|
def with_retries(retry_config = {})
|
24
|
-
|
25
|
-
|
26
|
-
|
27
|
-
|
43
|
+
@log_prefix = retry_config.delete(:log_prefix)
|
44
|
+
|
45
|
+
retry_config = @global_retry_config.dup.merge!(retry_config)
|
46
|
+
|
47
|
+
Retriable.retriable(retry_config) do
|
48
|
+
# Using begin -> rescue -> end for Ruby 2.4 compatibility
|
49
|
+
begin
|
50
|
+
if @reconnect_with_consistent_host
|
51
|
+
raise Lhm::Error.new("Could not reconnected to initial MySQL host. Aborting to avoid data-loss") unless same_host_as_initial?
|
52
|
+
end
|
53
|
+
|
54
|
+
yield(@connection)
|
55
|
+
rescue StandardError => e
|
56
|
+
# Not all errors should trigger a reconnect. Some errors such be raised and abort the LHM (such as reconnecting to the wrong host).
|
57
|
+
# The error will be raised the connection is still active (i.e. no need to reconnect) or if the connection is
|
58
|
+
# dead (i.e. not active) and @reconnect_with_host is false (i.e. instructed not to reconnect)
|
59
|
+
raise e if @connection.active? || (!@connection.active? && !@reconnect_with_consistent_host)
|
60
|
+
reconnect_with_host_check! if @reconnect_with_consistent_host
|
61
|
+
end
|
28
62
|
end
|
29
63
|
end
|
30
64
|
|
31
65
|
attr_reader :global_retry_config
|
66
|
+
attr_accessor :reconnect_with_consistent_host
|
32
67
|
|
33
68
|
private
|
34
69
|
|
70
|
+
def hostname
|
71
|
+
mysql_single_value(MYSQL_VAR_NAMES[:hostname])
|
72
|
+
end
|
73
|
+
|
74
|
+
def server_id
|
75
|
+
mysql_single_value(MYSQL_VAR_NAMES[:server_id])
|
76
|
+
end
|
77
|
+
|
78
|
+
def cloudsql?
|
79
|
+
mysql_single_value(MYSQL_VAR_NAMES[:version_comment]).include?(CLOUDSQL_VERSION_COMMENT)
|
80
|
+
end
|
81
|
+
|
82
|
+
def mysql_single_value(name)
|
83
|
+
query = Lhm::ProxySQLHelper.tagged("SELECT #{name} LIMIT 1")
|
84
|
+
|
85
|
+
@connection.execute(query).to_a.first.tap do |record|
|
86
|
+
return record&.first
|
87
|
+
end
|
88
|
+
end
|
89
|
+
|
90
|
+
def same_host_as_initial?
|
91
|
+
return @initial_server_id == server_id if cloudsql?
|
92
|
+
@initial_hostname == hostname
|
93
|
+
end
|
94
|
+
|
35
95
|
def log_with_prefix(message, level = :info)
|
36
96
|
message.prepend("[#{@log_prefix}] ") if @log_prefix
|
37
|
-
Lhm.logger.
|
97
|
+
Lhm.logger.public_send(level, message)
|
98
|
+
end
|
99
|
+
|
100
|
+
def reconnect_with_host_check!
|
101
|
+
log_with_prefix("Lost connection to MySQL, will retry to connect to same host")
|
102
|
+
begin
|
103
|
+
Retriable.retriable(host_retry_config) do
|
104
|
+
# tries to reconnect. On failure will trigger a retry
|
105
|
+
@connection.reconnect!
|
106
|
+
|
107
|
+
if same_host_as_initial?
|
108
|
+
# This is not an actual error, but controlled way to get the parent `Retriable.retriable` to retry
|
109
|
+
# the statement that failed (since the Retriable gem only retries on errors).
|
110
|
+
raise ReconnectToHostSuccessful.new("LHM successfully reconnected to initial host: #{@initial_hostname} (server_id: #{@initial_server_id})")
|
111
|
+
else
|
112
|
+
# New Master --> abort LHM (reconnecting will not change anything)
|
113
|
+
raise Lhm::Error.new("Reconnected to wrong host. Started migration on: #{@initial_hostname} (server_id: #{@initial_server_id}), but reconnected to: #{hostname} (server_id: #{@initial_server_id}).")
|
114
|
+
end
|
115
|
+
end
|
116
|
+
rescue StandardError => e
|
117
|
+
# The parent Retriable.retriable is configured to retry if it encounters an error with the success message.
|
118
|
+
# Therefore, if the connection is re-established successfully AND the host is the same, LHM can retry the query
|
119
|
+
# that originally failed.
|
120
|
+
raise e if reconnect_successful?(e)
|
121
|
+
# If the connection was not successful, the parent retriable will raise any error that originated from the
|
122
|
+
# `@connection.reconnect!`
|
123
|
+
# Therefore, this error will cause the LHM to abort
|
124
|
+
raise Lhm::Error.new("LHM tried the reconnection procedure but failed. Latest error: #{e.message}")
|
125
|
+
end
|
126
|
+
end
|
127
|
+
|
128
|
+
def reconnect_successful?(e)
|
129
|
+
e.is_a?(ReconnectToHostSuccessful)
|
38
130
|
end
|
39
131
|
|
40
132
|
# For a full list of configuration options see https://github.com/kamui/retriable
|
@@ -49,6 +141,11 @@ module Lhm
|
|
49
141
|
/Lost connection to MySQL server during query/,
|
50
142
|
/Max connect timeout reached/,
|
51
143
|
/Unknown MySQL server host/,
|
144
|
+
/connection is locked to hostgroup/,
|
145
|
+
/The MySQL server is running with the --read-only option so it cannot execute this statement/,
|
146
|
+
],
|
147
|
+
ReconnectToHostSuccessful => [
|
148
|
+
/#{RECONNECT_SUCCESSFUL_MESSAGE}/
|
52
149
|
]
|
53
150
|
},
|
54
151
|
multiplier: 1, # each successive interval grows by this factor
|
@@ -57,8 +154,29 @@ module Lhm
|
|
57
154
|
rand_factor: 0, # percentage to randomize the next retry interval time
|
58
155
|
max_elapsed_time: Float::INFINITY, # max total time in seconds that code is allowed to keep being retried
|
59
156
|
on_retry: Proc.new do |exception, try_number, total_elapsed_time, next_interval|
|
60
|
-
|
61
|
-
|
157
|
+
if reconnect_successful?(exception)
|
158
|
+
log_with_prefix("#{exception.message} -- triggering retry", :info)
|
159
|
+
else
|
160
|
+
log_with_prefix("#{exception.class}: '#{exception.message}' - #{try_number} tries in #{total_elapsed_time} seconds and #{next_interval} seconds until the next try.", :error)
|
161
|
+
end
|
162
|
+
end
|
163
|
+
}.freeze
|
164
|
+
end
|
165
|
+
|
166
|
+
def host_retry_config
|
167
|
+
{
|
168
|
+
on: {
|
169
|
+
StandardError => [
|
170
|
+
/Lost connection to MySQL server at 'reading initial communication packet'/
|
171
|
+
]
|
172
|
+
},
|
173
|
+
multiplier: 1, # each successive interval grows by this factor
|
174
|
+
base_interval: 0.25, # the initial interval in seconds between tries.
|
175
|
+
tries: 20, # Number of attempts to make at running your code block (includes initial attempt).
|
176
|
+
rand_factor: 0, # percentage to randomize the next retry interval time
|
177
|
+
max_elapsed_time: Float::INFINITY, # max total time in seconds that code is allowed to keep being retried
|
178
|
+
on_retry: Proc.new do |exception, try_number, total_elapsed_time, next_interval|
|
179
|
+
log_with_prefix("#{exception.class}: '#{exception.message}' - #{try_number} tries in #{total_elapsed_time} seconds and #{next_interval} seconds until the next try.", :error)
|
62
180
|
end
|
63
181
|
}.freeze
|
64
182
|
end
|
@@ -124,8 +124,8 @@ module Lhm
|
|
124
124
|
else
|
125
125
|
raise ArgumentError, "Expected #{config_proc.inspect} to respond to `call`"
|
126
126
|
end
|
127
|
-
else
|
128
|
-
|
127
|
+
else
|
128
|
+
db_config
|
129
129
|
end
|
130
130
|
config.deep_symbolize_keys!
|
131
131
|
config[:host] = @host
|
@@ -140,6 +140,23 @@ module Lhm
|
|
140
140
|
[nil]
|
141
141
|
end
|
142
142
|
end
|
143
|
+
|
144
|
+
private
|
145
|
+
|
146
|
+
def db_config
|
147
|
+
if ar_supports_db_config?
|
148
|
+
ActiveRecord::Base.connection_pool.db_config.configuration_hash.dup
|
149
|
+
else
|
150
|
+
ActiveRecord::Base.connection_pool.spec.config.dup
|
151
|
+
end
|
152
|
+
end
|
153
|
+
|
154
|
+
def ar_supports_db_config?
|
155
|
+
# https://api.rubyonrails.org/v6.0/classes/ActiveRecord/ConnectionAdapters/ConnectionPool.html <-- has spec
|
156
|
+
# vs
|
157
|
+
# https://api.rubyonrails.org/v6.1/classes/ActiveRecord/ConnectionAdapters/ConnectionPool.html <-- has db_config
|
158
|
+
ActiveRecord::VERSION::MAJOR > 6 || ActiveRecord::VERSION::MAJOR == 6 && ActiveRecord::VERSION::MINOR >= 1
|
159
|
+
end
|
143
160
|
end
|
144
161
|
end
|
145
162
|
end
|
data/lib/lhm/version.rb
CHANGED
data/lib/lhm.rb
CHANGED
@@ -8,6 +8,7 @@ require 'lhm/throttler'
|
|
8
8
|
require 'lhm/version'
|
9
9
|
require 'lhm/cleanup/current'
|
10
10
|
require 'lhm/sql_retry'
|
11
|
+
require 'lhm/proxysql_helper'
|
11
12
|
require 'lhm/connection'
|
12
13
|
require 'lhm/test_support'
|
13
14
|
require 'lhm/railtie' if defined?(Rails::Railtie)
|
@@ -82,16 +83,29 @@ module Lhm
|
|
82
83
|
Lhm::Cleanup::Current.new(run, table_name, connection, options).execute
|
83
84
|
end
|
84
85
|
|
85
|
-
|
86
|
-
|
86
|
+
# Setups DB connection
|
87
|
+
#
|
88
|
+
# @param [ActiveRecord::Base] connection ActiveRecord Connection
|
89
|
+
# @param [Hash] connection_options Optional options (defaults to: empty hash)
|
90
|
+
# @option connection_options [Boolean] :reconnect_with_consistent_host
|
91
|
+
# Active / Deactivate ProxySQL-aware reconnection procedure (default to: false)
|
92
|
+
def setup(connection, connection_options = {})
|
93
|
+
@@connection = Connection.new(connection: connection, options: connection_options)
|
87
94
|
end
|
88
95
|
|
89
|
-
|
90
|
-
|
91
|
-
|
92
|
-
|
93
|
-
|
94
|
-
|
96
|
+
# Setups DB connection
|
97
|
+
#
|
98
|
+
# @param [Hash] connection_options Optional options (defaults to: empty hash)
|
99
|
+
# @option connection_options [Boolean] :reconnect_with_consistent_host
|
100
|
+
# Active / Deactivate ProxySQL-aware reconnection procedure (default to: false)
|
101
|
+
def connection(connection_options = nil)
|
102
|
+
if @@connection.nil?
|
103
|
+
raise 'Please call Lhm.setup' unless defined?(ActiveRecord)
|
104
|
+
@@connection = Connection.new(connection: ActiveRecord::Base.connection, options: connection_options || {})
|
105
|
+
else
|
106
|
+
@@connection.options = connection_options unless connection_options.nil?
|
107
|
+
end
|
108
|
+
@@connection
|
95
109
|
end
|
96
110
|
|
97
111
|
def self.logger=(new_logger)
|
@@ -1,3 +1,6 @@
|
|
1
1
|
# Creates replication user in Writer
|
2
|
+
CREATE USER IF NOT EXISTS 'writer'@'%' IDENTIFIED BY 'password';
|
3
|
+
CREATE USER IF NOT EXISTS 'reader'@'%' IDENTIFIED BY 'password';
|
4
|
+
|
2
5
|
CREATE USER IF NOT EXISTS 'replication'@'%' IDENTIFIED BY 'password';
|
3
6
|
GRANT REPLICATION SLAVE ON *.* TO' replication'@'%' IDENTIFIED BY 'password';
|
@@ -16,9 +16,9 @@ describe Lhm::AtomicSwitcher do
|
|
16
16
|
describe 'switching' do
|
17
17
|
before(:each) do
|
18
18
|
Thread.abort_on_exception = true
|
19
|
-
@origin
|
19
|
+
@origin = table_create('origin')
|
20
20
|
@destination = table_create('destination')
|
21
|
-
@migration
|
21
|
+
@migration = Lhm::Migration.new(@origin, @destination)
|
22
22
|
@logs = StringIO.new
|
23
23
|
Lhm.logger = Logger.new(@logs)
|
24
24
|
@connection.execute('SET GLOBAL innodb_lock_wait_timeout=3')
|
@@ -32,11 +32,17 @@ describe Lhm::AtomicSwitcher do
|
|
32
32
|
it 'should retry and log on lock wait timeouts' do
|
33
33
|
ar_connection = mock()
|
34
34
|
ar_connection.stubs(:data_source_exists?).returns(true)
|
35
|
-
ar_connection.stubs(:
|
35
|
+
ar_connection.stubs(:active?).returns(true)
|
36
|
+
ar_connection.stubs(:execute).returns([["dummy"]], [["dummy"]], [["dummy"]])
|
37
|
+
.then
|
38
|
+
.raises(ActiveRecord::StatementInvalid, 'Lock wait timeout exceeded; try restarting transaction.')
|
39
|
+
.then
|
40
|
+
.returns([["dummy"]]) # Matches initial host -> triggers retry
|
41
|
+
|
42
|
+
connection = Lhm::Connection.new(connection: ar_connection, options: {reconnect_with_consistent_host: true})
|
36
43
|
|
37
|
-
connection = Lhm::Connection.new(connection: ar_connection)
|
38
44
|
|
39
|
-
switcher = Lhm::AtomicSwitcher.new(@migration, connection, retriable: {base_interval: 0})
|
45
|
+
switcher = Lhm::AtomicSwitcher.new(@migration, connection, retriable: { tries: 3, base_interval: 0 })
|
40
46
|
|
41
47
|
assert switcher.run
|
42
48
|
|
@@ -50,11 +56,22 @@ describe Lhm::AtomicSwitcher do
|
|
50
56
|
it 'should give up on lock wait timeouts after a configured number of tries' do
|
51
57
|
ar_connection = mock()
|
52
58
|
ar_connection.stubs(:data_source_exists?).returns(true)
|
53
|
-
ar_connection.stubs(:
|
54
|
-
|
55
|
-
|
56
|
-
|
57
|
-
|
59
|
+
ar_connection.stubs(:active?).returns(true)
|
60
|
+
ar_connection.stubs(:execute).returns([["dummy"]], [["dummy"]], [["dummy"]])
|
61
|
+
.then
|
62
|
+
.raises(ActiveRecord::StatementInvalid, 'Lock wait timeout exceeded; try restarting transaction.')
|
63
|
+
.then
|
64
|
+
.returns([["dummy"]]) # triggers retry 1
|
65
|
+
.then
|
66
|
+
.raises(ActiveRecord::StatementInvalid, 'Lock wait timeout exceeded; try restarting transaction.')
|
67
|
+
.then
|
68
|
+
.returns([["dummy"]]) # triggers retry 2
|
69
|
+
.then
|
70
|
+
.raises(ActiveRecord::StatementInvalid, 'Lock wait timeout exceeded; try restarting transaction.') # triggers retry 2
|
71
|
+
|
72
|
+
connection = Lhm::Connection.new(connection: ar_connection, options: {reconnect_with_consistent_host: true})
|
73
|
+
|
74
|
+
switcher = Lhm::AtomicSwitcher.new(@migration, connection, retriable: { tries: 2, base_interval: 0 })
|
58
75
|
|
59
76
|
assert_raises(ActiveRecord::StatementInvalid) { switcher.run }
|
60
77
|
end
|
@@ -11,7 +11,8 @@ describe Lhm::ChunkInsert do
|
|
11
11
|
@destination = table_create(:destination)
|
12
12
|
@migration = Lhm::Migration.new(@origin, @destination)
|
13
13
|
execute("insert into origin set id = 1001")
|
14
|
-
@
|
14
|
+
@connection = Lhm::Connection.new(connection: connection)
|
15
|
+
@instance = Lhm::ChunkInsert.new(@migration, @connection, 1001, 1001)
|
15
16
|
end
|
16
17
|
|
17
18
|
it "returns the count" do
|
@@ -223,7 +223,7 @@ describe Lhm::Chunker do
|
|
223
223
|
|
224
224
|
if master_slave_mode?
|
225
225
|
def throttler.slave_connection(slave)
|
226
|
-
config = ActiveRecord::Base.connection_pool.
|
226
|
+
config = ActiveRecord::Base.connection_pool.db_config.configuration_hash.dup
|
227
227
|
config[:host] = slave
|
228
228
|
config[:port] = 3307
|
229
229
|
ActiveRecord::Base.send('mysql2_connection', config)
|
@@ -6,6 +6,7 @@ require File.expand_path(File.dirname(__FILE__)) + '/integration_helper'
|
|
6
6
|
require 'lhm/table'
|
7
7
|
require 'lhm/migration'
|
8
8
|
require 'lhm/entangler'
|
9
|
+
require 'lhm/connection'
|
9
10
|
|
10
11
|
describe Lhm::Entangler do
|
11
12
|
include IntegrationHelper
|
@@ -17,7 +18,8 @@ describe Lhm::Entangler do
|
|
17
18
|
@origin = table_create('origin')
|
18
19
|
@destination = table_create('destination')
|
19
20
|
@migration = Lhm::Migration.new(@origin, @destination)
|
20
|
-
@
|
21
|
+
@connection = Lhm::Connection.new(connection: connection)
|
22
|
+
@entangler = Lhm::Entangler.new(@migration, @connection)
|
21
23
|
end
|
22
24
|
|
23
25
|
it 'should replay inserts from origin into destination' do
|
@@ -35,6 +35,15 @@ module IntegrationHelper
|
|
35
35
|
@connection
|
36
36
|
end
|
37
37
|
|
38
|
+
def connect_proxysql!
|
39
|
+
connect!(
|
40
|
+
'127.0.0.1',
|
41
|
+
$db_config['proxysql']['port'],
|
42
|
+
$db_config['proxysql']['user'],
|
43
|
+
$db_config['proxysql']['password'],
|
44
|
+
)
|
45
|
+
end
|
46
|
+
|
38
47
|
def connect_master!
|
39
48
|
connect!(
|
40
49
|
'127.0.0.1',
|
@@ -53,14 +62,23 @@ module IntegrationHelper
|
|
53
62
|
)
|
54
63
|
end
|
55
64
|
|
56
|
-
def
|
57
|
-
|
58
|
-
|
65
|
+
def connect_master_with_toxiproxy!(with_retry: false)
|
66
|
+
connect!(
|
67
|
+
'127.0.0.1',
|
68
|
+
$db_config['master_toxic']['port'],
|
69
|
+
$db_config['master_toxic']['user'],
|
70
|
+
$db_config['master_toxic']['password'],
|
71
|
+
with_retry)
|
72
|
+
end
|
73
|
+
|
74
|
+
def connect!(hostname, port, user, password, with_retry = false)
|
75
|
+
adapter = ar_conn(hostname, port, user, password)
|
76
|
+
Lhm.setup(adapter,{reconnect_with_consistent_host: with_retry})
|
59
77
|
unless defined?(@@cleaned_up)
|
60
78
|
Lhm.cleanup(true)
|
61
79
|
@@cleaned_up = true
|
62
80
|
end
|
63
|
-
@connection =
|
81
|
+
@connection = Lhm.connection
|
64
82
|
end
|
65
83
|
|
66
84
|
def ar_conn(host, port, user, password)
|
@@ -119,7 +137,7 @@ module IntegrationHelper
|
|
119
137
|
# Helps testing behaviour when another client locks the db
|
120
138
|
def start_locking_thread(lock_for, queue, locking_query)
|
121
139
|
Thread.new do
|
122
|
-
conn =
|
140
|
+
conn = new_mysql_connection
|
123
141
|
conn.query('BEGIN')
|
124
142
|
conn.query(locking_query)
|
125
143
|
queue.push(true)
|