lhm-shopify 4.0.0 → 4.1.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.github/workflows/test.yml +20 -18
- data/Appraisals +5 -11
- data/CHANGELOG.md +6 -0
- data/Gemfile.lock +22 -7
- data/README.md +7 -7
- data/dev.yml +4 -1
- data/docker-compose-mysql-5.7.yml +1 -0
- data/docker-compose-mysql-8.0.yml +63 -0
- data/docker-compose.yml +3 -3
- data/gemfiles/activerecord_6.1.gemfile +1 -0
- data/gemfiles/activerecord_6.1.gemfile.lock +8 -2
- data/gemfiles/activerecord_7.0.gemfile +1 -0
- data/gemfiles/activerecord_7.0.gemfile.lock +7 -1
- data/gemfiles/{activerecord_6.0.gemfile → activerecord_7.1.gemfile} +1 -1
- data/gemfiles/{activerecord_7.1.0.beta1.gemfile.lock → activerecord_7.1.gemfile.lock} +10 -8
- data/lhm.gemspec +1 -0
- data/lib/lhm/atomic_switcher.rb +3 -3
- data/lib/lhm/chunker.rb +4 -4
- data/lib/lhm/connection.rb +9 -1
- data/lib/lhm/sql_retry.rb +36 -18
- data/lib/lhm/table.rb +3 -4
- data/lib/lhm/throttler/replica_lag.rb +17 -13
- data/lib/lhm/version.rb +1 -1
- data/scripts/helpers/wait-for-dbs.sh +3 -3
- data/scripts/mysql/writer/create_users.sql +1 -1
- data/spec/integration/atomic_switcher_spec.rb +4 -8
- data/spec/integration/chunker_spec.rb +21 -6
- data/spec/integration/database.yml +3 -3
- data/spec/integration/integration_helper.rb +11 -3
- data/spec/integration/lhm_spec.rb +29 -13
- data/spec/integration/proxysql_spec.rb +10 -10
- data/spec/integration/sql_retry/db_connection_helper.rb +2 -4
- data/spec/integration/sql_retry/lock_wait_spec.rb +7 -8
- data/spec/integration/sql_retry/lock_wait_timeout_test_helper.rb +18 -10
- data/spec/integration/sql_retry/proxysql_helper.rb +1 -1
- data/spec/integration/sql_retry/retry_with_proxysql_spec.rb +1 -2
- data/spec/integration/table_spec.rb +1 -1
- data/spec/test_helper.rb +27 -3
- data/spec/unit/atomic_switcher_spec.rb +2 -2
- data/spec/unit/chunker_spec.rb +43 -43
- data/spec/unit/connection_spec.rb +2 -2
- data/spec/unit/entangler_spec.rb +14 -24
- data/spec/unit/throttler/replica_lag_spec.rb +6 -14
- metadata +21 -8
- data/.travis.yml +0 -21
- data/gemfiles/activerecord_6.0.gemfile.lock +0 -71
- data/gemfiles/activerecord_7.1.0.beta1.gemfile +0 -7
data/lib/lhm/table.rb
CHANGED
@@ -39,10 +39,9 @@ module Lhm
|
|
39
39
|
end
|
40
40
|
|
41
41
|
def ddl
|
42
|
-
|
43
|
-
|
44
|
-
@connection.
|
45
|
-
specification
|
42
|
+
query = "SHOW CREATE TABLE #{ @connection.quote_table_name(@table_name) }"
|
43
|
+
|
44
|
+
@connection.select_one(query)["Create Table"]
|
46
45
|
end
|
47
46
|
|
48
47
|
def parse
|
@@ -91,6 +91,14 @@ module Lhm
|
|
91
91
|
|
92
92
|
attr_reader :host, :connection
|
93
93
|
|
94
|
+
def self.client
|
95
|
+
defined?(Mysql2::Client) ? Mysql2::Client : Trilogy
|
96
|
+
end
|
97
|
+
|
98
|
+
def self.client_error
|
99
|
+
defined?(Mysql2::Error) ? Mysql2::Error : Trilogy::Error
|
100
|
+
end
|
101
|
+
|
94
102
|
def initialize(host, connection_config = nil)
|
95
103
|
@host = host
|
96
104
|
@connection_config = prepare_connection_config(connection_config)
|
@@ -108,13 +116,11 @@ module Lhm
|
|
108
116
|
private
|
109
117
|
|
110
118
|
def client(config)
|
111
|
-
|
112
|
-
|
113
|
-
|
114
|
-
|
115
|
-
|
116
|
-
nil
|
117
|
-
end
|
119
|
+
Lhm.logger.info "Connecting to #{@host} on database: #{config[:database]}"
|
120
|
+
self.class.client.new(config)
|
121
|
+
rescue self.class.client_error => e
|
122
|
+
Lhm.logger.info "Error connecting to #{@host}: #{e}"
|
123
|
+
nil
|
118
124
|
end
|
119
125
|
|
120
126
|
def prepare_connection_config(config_proc)
|
@@ -133,12 +139,10 @@ module Lhm
|
|
133
139
|
end
|
134
140
|
|
135
141
|
def query_connection(query, result)
|
136
|
-
|
137
|
-
|
138
|
-
|
139
|
-
|
140
|
-
[nil]
|
141
|
-
end
|
142
|
+
@connection.query(query).map { |row| row[result] }
|
143
|
+
rescue self.class.client_error => e
|
144
|
+
Lhm.logger.info "Unable to connect and/or query #{host}: #{e}"
|
145
|
+
[nil]
|
142
146
|
end
|
143
147
|
|
144
148
|
private
|
data/lib/lhm/version.rb
CHANGED
@@ -1,19 +1,19 @@
|
|
1
1
|
#!/bin/bash
|
2
2
|
# Wait for writer
|
3
3
|
echo "Waiting for MySQL-1: "
|
4
|
-
while ! (mysqladmin ping --host="127.0.0.1" --port=
|
4
|
+
while ! (mysqladmin ping --host="127.0.0.1" --port=13006 --user=root --password=password --silent 2> /dev/null); do
|
5
5
|
echo -ne "."
|
6
6
|
sleep 1
|
7
7
|
done
|
8
8
|
# Wait for reader
|
9
9
|
echo "Waiting for MySQL-2: "
|
10
|
-
while ! (mysqladmin ping --host="127.0.0.1" --port=
|
10
|
+
while ! (mysqladmin ping --host="127.0.0.1" --port=13007 --user=root --password=password --silent 2> /dev/null); do
|
11
11
|
echo -ne "."
|
12
12
|
sleep 1
|
13
13
|
done
|
14
14
|
# Wait for proxysql
|
15
15
|
echo "Waiting for ProxySQL:"
|
16
|
-
while ! (mysqladmin ping --host="127.0.0.1" --port=
|
16
|
+
while ! (mysqladmin ping --host="127.0.0.1" --port=13005 --user=root --password=password --silent 2> /dev/null); do
|
17
17
|
echo -ne "."
|
18
18
|
sleep 1
|
19
19
|
done
|
@@ -3,4 +3,4 @@ CREATE USER IF NOT EXISTS 'writer'@'%' IDENTIFIED BY 'password';
|
|
3
3
|
CREATE USER IF NOT EXISTS 'reader'@'%' IDENTIFIED BY 'password';
|
4
4
|
|
5
5
|
CREATE USER IF NOT EXISTS 'replication'@'%' IDENTIFIED BY 'password';
|
6
|
-
GRANT REPLICATION SLAVE ON *.* TO' replication'@'%'
|
6
|
+
GRANT REPLICATION SLAVE ON *.* TO' replication'@'%';
|
@@ -33,8 +33,8 @@ describe Lhm::AtomicSwitcher do
|
|
33
33
|
ar_connection = mock()
|
34
34
|
ar_connection.stubs(:data_source_exists?).returns(true)
|
35
35
|
ar_connection.stubs(:active?).returns(true)
|
36
|
-
ar_connection.stubs(:
|
37
|
-
|
36
|
+
ar_connection.stubs(:select_value).returns("dummy")
|
37
|
+
ar_connection.stubs(:execute)
|
38
38
|
.raises(ActiveRecord::StatementInvalid, 'Lock wait timeout exceeded; try restarting transaction.')
|
39
39
|
.then
|
40
40
|
.returns([["dummy"]]) # Matches initial host -> triggers retry
|
@@ -62,16 +62,12 @@ describe Lhm::AtomicSwitcher do
|
|
62
62
|
ar_connection = mock()
|
63
63
|
ar_connection.stubs(:data_source_exists?).returns(true)
|
64
64
|
ar_connection.stubs(:active?).returns(true)
|
65
|
-
ar_connection.stubs(:
|
66
|
-
|
65
|
+
ar_connection.stubs(:select_value).returns("dummy")
|
66
|
+
ar_connection.stubs(:execute)
|
67
67
|
.raises(ActiveRecord::StatementInvalid, 'Lock wait timeout exceeded; try restarting transaction.')
|
68
68
|
.then
|
69
|
-
.returns([["dummy"]]) # triggers retry 1
|
70
|
-
.then
|
71
69
|
.raises(ActiveRecord::StatementInvalid, 'Lock wait timeout exceeded; try restarting transaction.')
|
72
70
|
.then
|
73
|
-
.returns([["dummy"]]) # triggers retry 2
|
74
|
-
.then
|
75
71
|
.raises(ActiveRecord::StatementInvalid, 'Lock wait timeout exceeded; try restarting transaction.') # triggers retry 2
|
76
72
|
|
77
73
|
connection = Lhm::Connection.new(connection: ar_connection, options: {
|
@@ -74,7 +74,8 @@ describe Lhm::Chunker do
|
|
74
74
|
Lhm::Chunker.new(migration, connection, {raise_on_warnings: true, throttler: throttler, printer: printer} ).run
|
75
75
|
end
|
76
76
|
|
77
|
-
|
77
|
+
error_key = index_key("custom_primary_key_dest", "index_custom_primary_key_on_id")
|
78
|
+
assert_match "Unexpected warning found for inserted row: Duplicate entry '1001' for key '#{error_key}'", exception.message
|
78
79
|
end
|
79
80
|
|
80
81
|
it 'should copy and warn on unexpected warnings by default' do
|
@@ -87,8 +88,10 @@ describe Lhm::Chunker do
|
|
87
88
|
|
88
89
|
Lhm::Chunker.new(migration, connection, {throttler: throttler, printer: printer} ).run
|
89
90
|
|
91
|
+
error_key = index_key("custom_primary_key_dest", "index_custom_primary_key_on_id")
|
92
|
+
|
90
93
|
assert_equal 2, log_messages.length
|
91
|
-
assert log_messages[1].include?("Unexpected warning found for inserted row: Duplicate entry '1001' for key '
|
94
|
+
assert log_messages[1].include?("Unexpected warning found for inserted row: Duplicate entry '1001' for key '#{error_key}'"), log_messages
|
92
95
|
end
|
93
96
|
|
94
97
|
it 'should log two times for two unexpected warnings' do
|
@@ -103,9 +106,11 @@ describe Lhm::Chunker do
|
|
103
106
|
|
104
107
|
Lhm::Chunker.new(migration, connection, {throttler: throttler, printer: printer} ).run
|
105
108
|
|
109
|
+
error_key = index_key("custom_primary_key_dest", "index_custom_primary_key_on_id")
|
110
|
+
|
106
111
|
assert_equal 3, log_messages.length
|
107
|
-
assert log_messages[1].include?("Unexpected warning found for inserted row: Duplicate entry '1001' for key '
|
108
|
-
assert log_messages[2].include?("Unexpected warning found for inserted row: Duplicate entry '1002' for key '
|
112
|
+
assert log_messages[1].include?("Unexpected warning found for inserted row: Duplicate entry '1001' for key '#{error_key}'"), log_messages
|
113
|
+
assert log_messages[2].include?("Unexpected warning found for inserted row: Duplicate entry '1002' for key '#{error_key}'"), log_messages
|
109
114
|
end
|
110
115
|
|
111
116
|
it 'should copy and warn on unexpected warnings' do
|
@@ -118,8 +123,10 @@ describe Lhm::Chunker do
|
|
118
123
|
|
119
124
|
Lhm::Chunker.new(migration, connection, {raise_on_warnings: false, throttler: throttler, printer: printer} ).run
|
120
125
|
|
126
|
+
error_key = index_key("custom_primary_key_dest", "index_custom_primary_key_on_id")
|
127
|
+
|
121
128
|
assert_equal 2, log_messages.length
|
122
|
-
assert log_messages[1].include?("Unexpected warning found for inserted row: Duplicate entry '1001' for key '
|
129
|
+
assert log_messages[1].include?("Unexpected warning found for inserted row: Duplicate entry '1001' for key '#{error_key}'"), log_messages
|
123
130
|
end
|
124
131
|
|
125
132
|
it 'should create the modified destination, even if the source is empty' do
|
@@ -222,7 +229,7 @@ describe Lhm::Chunker do
|
|
222
229
|
def throttler.replica_connection(replica)
|
223
230
|
config = ActiveRecord::Base.connection_pool.db_config.configuration_hash.dup
|
224
231
|
config[:host] = replica
|
225
|
-
config[:port] =
|
232
|
+
config[:port] = 13007
|
226
233
|
ActiveRecord::Base.send('mysql2_connection', config)
|
227
234
|
end
|
228
235
|
end
|
@@ -261,4 +268,12 @@ describe Lhm::Chunker do
|
|
261
268
|
end
|
262
269
|
end
|
263
270
|
end
|
271
|
+
|
272
|
+
def index_key(table_name, index_name)
|
273
|
+
if mysql_version.start_with?("8")
|
274
|
+
"#{table_name}.#{index_name}"
|
275
|
+
else
|
276
|
+
index_name
|
277
|
+
end
|
278
|
+
end
|
264
279
|
end
|
@@ -2,17 +2,17 @@ master:
|
|
2
2
|
host: mysql-1
|
3
3
|
user: root
|
4
4
|
password: password
|
5
|
-
port:
|
5
|
+
port: 13006
|
6
6
|
replica:
|
7
7
|
host: mysql-2
|
8
8
|
user: root
|
9
9
|
password: password
|
10
|
-
port:
|
10
|
+
port: 13007
|
11
11
|
proxysql:
|
12
12
|
host: proxysql
|
13
13
|
user: root
|
14
14
|
password: password
|
15
|
-
port:
|
15
|
+
port: 13005
|
16
16
|
master_toxic:
|
17
17
|
host: toxiproxy
|
18
18
|
user: root
|
@@ -22,7 +22,7 @@ module IntegrationHelper
|
|
22
22
|
def self.included(base)
|
23
23
|
base.after(:each) do
|
24
24
|
cleanup_connection = new_mysql_connection
|
25
|
-
results =
|
25
|
+
results = DATABASE.query(cleanup_connection, "SELECT table_name FROM information_schema.tables WHERE table_schema = '#{$db_name}';")
|
26
26
|
table_names_for_cleanup = results.map { |row| "#{$db_name}." + row.values.first }
|
27
27
|
cleanup_connection.query("DROP TABLE IF EXISTS #{table_names_for_cleanup.join(', ')};") if table_names_for_cleanup.length > 0
|
28
28
|
end
|
@@ -35,6 +35,14 @@ module IntegrationHelper
|
|
35
35
|
@connection
|
36
36
|
end
|
37
37
|
|
38
|
+
def mysql_version
|
39
|
+
@mysql_version ||= begin
|
40
|
+
# This SQL returns a value of shape: X.Y.ZZ-AA-log
|
41
|
+
result = connection.query("SELECT VERSION()")
|
42
|
+
result.dig(0, 0).split("-", 2)[0]
|
43
|
+
end
|
44
|
+
end
|
45
|
+
|
38
46
|
def connect_proxysql!
|
39
47
|
connect!(
|
40
48
|
'127.0.0.1',
|
@@ -81,7 +89,7 @@ module IntegrationHelper
|
|
81
89
|
|
82
90
|
def ar_conn(host, port, user, password)
|
83
91
|
ActiveRecord::Base.establish_connection(
|
84
|
-
:adapter =>
|
92
|
+
:adapter => DATABASE.adapter,
|
85
93
|
:host => host,
|
86
94
|
:username => user,
|
87
95
|
:port => port,
|
@@ -171,7 +179,7 @@ module IntegrationHelper
|
|
171
179
|
end
|
172
180
|
|
173
181
|
def new_mysql_connection(role='master')
|
174
|
-
|
182
|
+
DATABASE.client.new(
|
175
183
|
host: '127.0.0.1',
|
176
184
|
database: $db_name,
|
177
185
|
username: $db_config[role]['user'],
|
@@ -9,6 +9,10 @@ describe Lhm do
|
|
9
9
|
|
10
10
|
before(:each) { connect_master!; Lhm.cleanup(true) }
|
11
11
|
|
12
|
+
let(:collation) do
|
13
|
+
mysql_version.start_with?("8.0") ? "utf8mb3_general_ci" : "utf8_general_ci"
|
14
|
+
end
|
15
|
+
|
12
16
|
describe 'id column requirement' do
|
13
17
|
it 'should migrate the table when id is pk' do
|
14
18
|
table_create(:users)
|
@@ -17,9 +21,11 @@ describe Lhm do
|
|
17
21
|
t.add_column(:logins, "int(12) default '0'")
|
18
22
|
end
|
19
23
|
|
24
|
+
expected_type = mysql_version.start_with?("8.0") ? "int" : "int(12)"
|
25
|
+
|
20
26
|
replica do
|
21
27
|
value(table_read(:users).columns['logins']).must_equal({
|
22
|
-
:type =>
|
28
|
+
:type => expected_type,
|
23
29
|
:is_nullable => 'YES',
|
24
30
|
:column_default => '0',
|
25
31
|
:comment => '',
|
@@ -35,9 +41,11 @@ describe Lhm do
|
|
35
41
|
t.add_column(:logins, "int(12) default '0'")
|
36
42
|
end
|
37
43
|
|
44
|
+
expected_type = mysql_version.start_with?("8.0") ? "int" : "int(12)"
|
45
|
+
|
38
46
|
replica do
|
39
47
|
value(table_read(:custom_primary_key).columns['logins']).must_equal({
|
40
|
-
:type =>
|
48
|
+
:type => expected_type,
|
41
49
|
:is_nullable => 'YES',
|
42
50
|
:column_default => '0',
|
43
51
|
:comment => '',
|
@@ -53,9 +61,11 @@ describe Lhm do
|
|
53
61
|
t.add_column(:logins, "int(12) default '0'")
|
54
62
|
end
|
55
63
|
|
64
|
+
expected_type = mysql_version.start_with?("8.0") ? "int" : "int(12)"
|
65
|
+
|
56
66
|
replica do
|
57
67
|
value(table_read(:composite_primary_key).columns['logins']).must_equal({
|
58
|
-
:type =>
|
68
|
+
:type => expected_type,
|
59
69
|
:is_nullable => 'YES',
|
60
70
|
:column_default => '0',
|
61
71
|
:comment => '',
|
@@ -132,9 +142,11 @@ describe Lhm do
|
|
132
142
|
t.add_column(:logins, "INT(12) DEFAULT '0'")
|
133
143
|
end
|
134
144
|
|
145
|
+
expected_type = mysql_version.start_with?("8.0") ? "int" : "int(12)"
|
146
|
+
|
135
147
|
replica do
|
136
148
|
value(table_read(:users).columns['logins']).must_equal({
|
137
|
-
:type =>
|
149
|
+
:type => expected_type,
|
138
150
|
:is_nullable => 'YES',
|
139
151
|
:column_default => '0',
|
140
152
|
:comment => '',
|
@@ -272,7 +284,7 @@ describe Lhm do
|
|
272
284
|
:is_nullable => 'NO',
|
273
285
|
:column_default => 'none',
|
274
286
|
:comment => '',
|
275
|
-
:collate =>
|
287
|
+
:collate => collation,
|
276
288
|
})
|
277
289
|
end
|
278
290
|
end
|
@@ -284,9 +296,11 @@ describe Lhm do
|
|
284
296
|
t.change_column(:id, 'int(5)')
|
285
297
|
end
|
286
298
|
|
299
|
+
expected_type = mysql_version.start_with?("8.0") ? "int" : "int(5)"
|
300
|
+
|
287
301
|
replica do
|
288
302
|
value(table_read(:small_table).columns['id']).must_equal({
|
289
|
-
:type =>
|
303
|
+
:type => expected_type,
|
290
304
|
:is_nullable => 'NO',
|
291
305
|
:column_default => nil,
|
292
306
|
:comment => '',
|
@@ -311,7 +325,7 @@ describe Lhm do
|
|
311
325
|
:is_nullable => 'YES',
|
312
326
|
:column_default => nil,
|
313
327
|
:comment => '',
|
314
|
-
:collate =>
|
328
|
+
:collate => collation,
|
315
329
|
})
|
316
330
|
|
317
331
|
result = select_one('SELECT login from users')
|
@@ -336,7 +350,7 @@ describe Lhm do
|
|
336
350
|
:is_nullable => 'YES',
|
337
351
|
:column_default => 'Superfriends',
|
338
352
|
:comment => '',
|
339
|
-
:collate =>
|
353
|
+
:collate => collation,
|
340
354
|
})
|
341
355
|
|
342
356
|
result = select_one('SELECT `fnord` from users')
|
@@ -383,11 +397,13 @@ describe Lhm do
|
|
383
397
|
t.rename_column(:reference, :ref)
|
384
398
|
end
|
385
399
|
|
400
|
+
expected_type = mysql_version.start_with?("8.0") ? "int" : "int(11)"
|
401
|
+
|
386
402
|
replica do
|
387
403
|
table_data = table_read(:users)
|
388
404
|
assert_nil table_data.columns['reference']
|
389
405
|
value(table_read(:users).columns['ref']).must_equal({
|
390
|
-
:type =>
|
406
|
+
:type => expected_type,
|
391
407
|
:is_nullable => 'YES',
|
392
408
|
:column_default => nil,
|
393
409
|
:comment => 'RefComment',
|
@@ -418,7 +434,7 @@ describe Lhm do
|
|
418
434
|
:is_nullable => 'YES',
|
419
435
|
:column_default => nil,
|
420
436
|
:comment => '',
|
421
|
-
:collate =>
|
437
|
+
:collate => collation,
|
422
438
|
})
|
423
439
|
|
424
440
|
result = select_one('SELECT `fnord` from users')
|
@@ -443,7 +459,7 @@ describe Lhm do
|
|
443
459
|
:is_nullable => 'YES',
|
444
460
|
:column_default => nil,
|
445
461
|
:comment => '',
|
446
|
-
:collate =>
|
462
|
+
:collate => collation,
|
447
463
|
})
|
448
464
|
|
449
465
|
result = select_one('SELECT `user_name` from users')
|
@@ -470,7 +486,7 @@ describe Lhm do
|
|
470
486
|
:is_nullable => 'NO',
|
471
487
|
:column_default => nil,
|
472
488
|
:comment => '',
|
473
|
-
:collate =>
|
489
|
+
:collate => collation,
|
474
490
|
})
|
475
491
|
|
476
492
|
result = select_one('SELECT `user_name` from users')
|
@@ -517,7 +533,7 @@ describe Lhm do
|
|
517
533
|
:is_nullable => 'YES',
|
518
534
|
:column_default => 'Superfriends',
|
519
535
|
:comment => '',
|
520
|
-
:collate =>
|
536
|
+
:collate => collation,
|
521
537
|
})
|
522
538
|
end
|
523
539
|
end
|
@@ -1,34 +1,34 @@
|
|
1
1
|
describe "ProxySQL integration" do
|
2
2
|
it "Should contact the writer" do
|
3
|
-
conn =
|
3
|
+
conn = DATABASE.client.new(
|
4
4
|
host: '127.0.0.1',
|
5
5
|
username: "writer",
|
6
6
|
password: "password",
|
7
|
-
port: "
|
7
|
+
port: "13005",
|
8
8
|
)
|
9
9
|
|
10
|
-
assert_equal
|
10
|
+
assert_equal DATABASE.query(conn, "SELECT @@global.hostname as host").each.first["host"], "mysql-1"
|
11
11
|
end
|
12
12
|
|
13
13
|
it "Should contact the reader" do
|
14
|
-
conn =
|
14
|
+
conn = DATABASE.client.new(
|
15
15
|
host: '127.0.0.1',
|
16
16
|
username: "reader",
|
17
17
|
password: "password",
|
18
|
-
port: "
|
18
|
+
port: "13005",
|
19
19
|
)
|
20
20
|
|
21
|
-
assert_equal
|
21
|
+
assert_equal DATABASE.query(conn, "SELECT @@global.hostname as host").each.first["host"], "mysql-2"
|
22
22
|
end
|
23
23
|
|
24
24
|
it "Should override default hostgroup from user if rule matches" do
|
25
|
-
conn =
|
25
|
+
conn = DATABASE.client.new(
|
26
26
|
host: '127.0.0.1',
|
27
27
|
username: "reader",
|
28
28
|
password: "password",
|
29
|
-
port: "
|
29
|
+
port: "13005",
|
30
30
|
)
|
31
31
|
|
32
|
-
assert_equal
|
32
|
+
assert_equal DATABASE.query(conn, "SELECT @@global.hostname as host #{Lhm::ProxySQLHelper::ANNOTATION}").each.first["host"], "mysql-1"
|
33
33
|
end
|
34
|
-
end
|
34
|
+
end
|
@@ -1,5 +1,4 @@
|
|
1
1
|
require 'yaml'
|
2
|
-
require 'mysql2'
|
3
2
|
|
4
3
|
class DBConnectionHelper
|
5
4
|
|
@@ -11,12 +10,11 @@ class DBConnectionHelper
|
|
11
10
|
end
|
12
11
|
|
13
12
|
def new_mysql_connection(role = :master, with_data = false, toxic = false)
|
14
|
-
|
15
13
|
key = role.to_s + toxic_postfix(toxic)
|
16
14
|
|
17
15
|
conn = ActiveRecord::Base.establish_connection(
|
18
16
|
:host => '127.0.0.1',
|
19
|
-
:adapter =>
|
17
|
+
:adapter => DATABASE.adapter,
|
20
18
|
:username => db_config[key]['user'],
|
21
19
|
:password => db_config[key]['password'],
|
22
20
|
:database => test_db_name,
|
@@ -49,4 +47,4 @@ class DBConnectionHelper
|
|
49
47
|
end
|
50
48
|
end
|
51
49
|
end
|
52
|
-
end
|
50
|
+
end
|
@@ -1,5 +1,4 @@
|
|
1
1
|
require 'minitest/autorun'
|
2
|
-
require 'mysql2'
|
3
2
|
require 'integration/sql_retry/lock_wait_timeout_test_helper'
|
4
3
|
require 'lhm'
|
5
4
|
|
@@ -22,7 +21,7 @@ describe Lhm::SqlRetry do
|
|
22
21
|
# Assert our pre-conditions
|
23
22
|
assert_equal 2, @helper.record_count
|
24
23
|
|
25
|
-
|
24
|
+
DATABASE.client.any_instance.stubs(:active?).returns(true)
|
26
25
|
end
|
27
26
|
|
28
27
|
after(:each) do
|
@@ -43,8 +42,8 @@ describe Lhm::SqlRetry do
|
|
43
42
|
|
44
43
|
exception = assert_raises { @helper.trigger_wait_lock }
|
45
44
|
|
46
|
-
assert_match
|
47
|
-
assert_equal
|
45
|
+
assert_match Regexp.new("Lock wait timeout exceeded; try restarting transaction"), exception.message
|
46
|
+
assert_equal DATABASE.timeout_error, exception.class
|
48
47
|
|
49
48
|
assert_equal 2, @helper.record_count # no records inserted
|
50
49
|
puts "*" * 64
|
@@ -82,10 +81,10 @@ describe Lhm::SqlRetry do
|
|
82
81
|
logs = @logger.string.split("\n")
|
83
82
|
assert_equal 2, logs.length
|
84
83
|
|
85
|
-
assert logs.first.include?("
|
84
|
+
assert logs.first.include?("Lock wait timeout exceeded; try restarting transaction' - 1 tries")
|
86
85
|
assert logs.first.include?("0.2 seconds until the next try")
|
87
86
|
|
88
|
-
assert logs.last.include?("
|
87
|
+
assert logs.last.include?("Lock wait timeout exceeded; try restarting transaction' - 2 tries")
|
89
88
|
assert logs.last.include?("0.2 seconds until the next try")
|
90
89
|
end
|
91
90
|
|
@@ -118,8 +117,8 @@ describe Lhm::SqlRetry do
|
|
118
117
|
|
119
118
|
exception = assert_raises { @helper.trigger_wait_lock }
|
120
119
|
|
121
|
-
assert_match
|
122
|
-
assert_equal
|
120
|
+
assert_match Regexp.new("Lock wait timeout exceeded; try restarting transaction"), exception.message
|
121
|
+
assert_equal DATABASE.timeout_error, exception.class
|
123
122
|
|
124
123
|
assert_equal 2, @helper.record_count # no records inserted
|
125
124
|
puts "*" * 64
|
@@ -14,13 +14,21 @@ class LockWaitTimeoutTestHelper
|
|
14
14
|
|
15
15
|
@lock_duration = lock_duration
|
16
16
|
|
17
|
-
# While implementing this, I discovered that MySQL seems to have an off-by-one
|
18
|
-
# bug with the innodb_lock_wait_timeout. If you ask it to wait 2 seconds, it will wait 3.
|
19
|
-
# In order to avoid surprisingly the user, let's account for that here, but also
|
20
|
-
# guard against a case where we go below 1, the minimum value.
|
21
|
-
raise ArgumentError, "innodb_lock_wait_timeout must be greater than or equal to 2" unless innodb_lock_wait_timeout >= 2
|
22
17
|
raise ArgumentError, "innodb_lock_wait_timeout must be an integer" if innodb_lock_wait_timeout.class != Integer
|
23
|
-
|
18
|
+
|
19
|
+
result = DATABASE.query(@main_conn, "SELECT VERSION()")
|
20
|
+
mysql_version = result.to_a.dig(0, "VERSION()").split("-", 2)[0]
|
21
|
+
|
22
|
+
if mysql_version.start_with?("8")
|
23
|
+
@innodb_lock_wait_timeout = innodb_lock_wait_timeout
|
24
|
+
else
|
25
|
+
# While implementing this, I discovered that MySQL seems to have an off-by-one
|
26
|
+
# bug with the innodb_lock_wait_timeout. If you ask it to wait 2 seconds, it will wait 3.
|
27
|
+
# In order to avoid surprisingly the user, let's account for that here, but also
|
28
|
+
# guard against a case where we go below 1, the minimum value.
|
29
|
+
raise ArgumentError, "innodb_lock_wait_timeout must be greater than or equal to 2" unless innodb_lock_wait_timeout >= 2
|
30
|
+
@innodb_lock_wait_timeout = innodb_lock_wait_timeout - 1
|
31
|
+
end
|
24
32
|
|
25
33
|
@threads = []
|
26
34
|
@queue = Queue.new
|
@@ -51,7 +59,7 @@ class LockWaitTimeoutTestHelper
|
|
51
59
|
end
|
52
60
|
|
53
61
|
def record_count(connection = main_conn)
|
54
|
-
response = connection
|
62
|
+
response = mysql_exec(connection, "SELECT COUNT(id) FROM #{test_table_name}")
|
55
63
|
response.first.values.first
|
56
64
|
end
|
57
65
|
|
@@ -79,7 +87,7 @@ class LockWaitTimeoutTestHelper
|
|
79
87
|
attr_reader :main_conn, :lock_duration, :innodb_lock_wait_timeout
|
80
88
|
|
81
89
|
def new_mysql_connection
|
82
|
-
|
90
|
+
DATABASE.client.new(
|
83
91
|
host: '127.0.0.1',
|
84
92
|
username: db_config['master']['user'],
|
85
93
|
password: db_config['master']['password'],
|
@@ -103,8 +111,8 @@ class LockWaitTimeoutTestHelper
|
|
103
111
|
private
|
104
112
|
|
105
113
|
def mysql_exec(connection, statement)
|
106
|
-
if connection.class ==
|
107
|
-
|
114
|
+
if connection.class == DATABASE.client
|
115
|
+
DATABASE.query(connection, statement)
|
108
116
|
elsif connection.class.to_s.include?("ActiveRecord")
|
109
117
|
connection.execute(statement)
|
110
118
|
else
|
@@ -2,7 +2,7 @@ class ProxySQLHelper
|
|
2
2
|
class << self
|
3
3
|
# Flips the destination hostgroup for /maintenance:lhm/ from 0 (i.e. writer) to 1 (i.e. reader)
|
4
4
|
def with_lhm_hostgroup_flip
|
5
|
-
conn =
|
5
|
+
conn = DATABASE.client.new(
|
6
6
|
host: '127.0.0.1',
|
7
7
|
username: "remote-admin",
|
8
8
|
password: "password",
|
@@ -1,5 +1,4 @@
|
|
1
1
|
require 'minitest/autorun'
|
2
|
-
require 'mysql2'
|
3
2
|
require 'lhm'
|
4
3
|
require 'toxiproxy'
|
5
4
|
|
@@ -50,7 +49,7 @@ describe Lhm::SqlRetry, "ProxiSQL tests for LHM retry" do
|
|
50
49
|
end
|
51
50
|
end
|
52
51
|
|
53
|
-
assert_equal @connection.
|
52
|
+
assert_equal 2000, @connection.select_one("SELECT * FROM #{DBConnectionHelper.test_table_name} WHERE id=2000")["id"]
|
54
53
|
|
55
54
|
logs = @logger.string.split("\n")
|
56
55
|
|
@@ -23,7 +23,7 @@ describe Lhm::Table do
|
|
23
23
|
end
|
24
24
|
|
25
25
|
it 'should parse columns' do
|
26
|
-
value(@table.columns['id'][:type]).must_match(/(bigint|int)\(\d+\)
|
26
|
+
value(@table.columns['id'][:type]).must_match(/(bigint|int)(\(\d+\))?/)
|
27
27
|
end
|
28
28
|
|
29
29
|
it 'should return true for method that should be renamed' do
|