lhm-shopify 4.0.0 → 4.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.github/workflows/test.yml +20 -18
- data/Appraisals +5 -11
- data/CHANGELOG.md +6 -0
- data/Gemfile.lock +22 -7
- data/README.md +7 -7
- data/dev.yml +4 -1
- data/docker-compose-mysql-5.7.yml +1 -0
- data/docker-compose-mysql-8.0.yml +63 -0
- data/docker-compose.yml +3 -3
- data/gemfiles/activerecord_6.1.gemfile +1 -0
- data/gemfiles/activerecord_6.1.gemfile.lock +8 -2
- data/gemfiles/activerecord_7.0.gemfile +1 -0
- data/gemfiles/activerecord_7.0.gemfile.lock +7 -1
- data/gemfiles/{activerecord_6.0.gemfile → activerecord_7.1.gemfile} +1 -1
- data/gemfiles/{activerecord_7.1.0.beta1.gemfile.lock → activerecord_7.1.gemfile.lock} +10 -8
- data/lhm.gemspec +1 -0
- data/lib/lhm/atomic_switcher.rb +3 -3
- data/lib/lhm/chunker.rb +4 -4
- data/lib/lhm/connection.rb +9 -1
- data/lib/lhm/sql_retry.rb +36 -18
- data/lib/lhm/table.rb +3 -4
- data/lib/lhm/throttler/replica_lag.rb +17 -13
- data/lib/lhm/version.rb +1 -1
- data/scripts/helpers/wait-for-dbs.sh +3 -3
- data/scripts/mysql/writer/create_users.sql +1 -1
- data/spec/integration/atomic_switcher_spec.rb +4 -8
- data/spec/integration/chunker_spec.rb +21 -6
- data/spec/integration/database.yml +3 -3
- data/spec/integration/integration_helper.rb +11 -3
- data/spec/integration/lhm_spec.rb +29 -13
- data/spec/integration/proxysql_spec.rb +10 -10
- data/spec/integration/sql_retry/db_connection_helper.rb +2 -4
- data/spec/integration/sql_retry/lock_wait_spec.rb +7 -8
- data/spec/integration/sql_retry/lock_wait_timeout_test_helper.rb +18 -10
- data/spec/integration/sql_retry/proxysql_helper.rb +1 -1
- data/spec/integration/sql_retry/retry_with_proxysql_spec.rb +1 -2
- data/spec/integration/table_spec.rb +1 -1
- data/spec/test_helper.rb +27 -3
- data/spec/unit/atomic_switcher_spec.rb +2 -2
- data/spec/unit/chunker_spec.rb +43 -43
- data/spec/unit/connection_spec.rb +2 -2
- data/spec/unit/entangler_spec.rb +14 -24
- data/spec/unit/throttler/replica_lag_spec.rb +6 -14
- metadata +21 -8
- data/.travis.yml +0 -21
- data/gemfiles/activerecord_6.0.gemfile.lock +0 -71
- data/gemfiles/activerecord_7.1.0.beta1.gemfile +0 -7
data/lib/lhm/table.rb
CHANGED
@@ -39,10 +39,9 @@ module Lhm
|
|
39
39
|
end
|
40
40
|
|
41
41
|
def ddl
|
42
|
-
|
43
|
-
|
44
|
-
@connection.
|
45
|
-
specification
|
42
|
+
query = "SHOW CREATE TABLE #{ @connection.quote_table_name(@table_name) }"
|
43
|
+
|
44
|
+
@connection.select_one(query)["Create Table"]
|
46
45
|
end
|
47
46
|
|
48
47
|
def parse
|
@@ -91,6 +91,14 @@ module Lhm
|
|
91
91
|
|
92
92
|
attr_reader :host, :connection
|
93
93
|
|
94
|
+
def self.client
|
95
|
+
defined?(Mysql2::Client) ? Mysql2::Client : Trilogy
|
96
|
+
end
|
97
|
+
|
98
|
+
def self.client_error
|
99
|
+
defined?(Mysql2::Error) ? Mysql2::Error : Trilogy::Error
|
100
|
+
end
|
101
|
+
|
94
102
|
def initialize(host, connection_config = nil)
|
95
103
|
@host = host
|
96
104
|
@connection_config = prepare_connection_config(connection_config)
|
@@ -108,13 +116,11 @@ module Lhm
|
|
108
116
|
private
|
109
117
|
|
110
118
|
def client(config)
|
111
|
-
|
112
|
-
|
113
|
-
|
114
|
-
|
115
|
-
|
116
|
-
nil
|
117
|
-
end
|
119
|
+
Lhm.logger.info "Connecting to #{@host} on database: #{config[:database]}"
|
120
|
+
self.class.client.new(config)
|
121
|
+
rescue self.class.client_error => e
|
122
|
+
Lhm.logger.info "Error connecting to #{@host}: #{e}"
|
123
|
+
nil
|
118
124
|
end
|
119
125
|
|
120
126
|
def prepare_connection_config(config_proc)
|
@@ -133,12 +139,10 @@ module Lhm
|
|
133
139
|
end
|
134
140
|
|
135
141
|
def query_connection(query, result)
|
136
|
-
|
137
|
-
|
138
|
-
|
139
|
-
|
140
|
-
[nil]
|
141
|
-
end
|
142
|
+
@connection.query(query).map { |row| row[result] }
|
143
|
+
rescue self.class.client_error => e
|
144
|
+
Lhm.logger.info "Unable to connect and/or query #{host}: #{e}"
|
145
|
+
[nil]
|
142
146
|
end
|
143
147
|
|
144
148
|
private
|
data/lib/lhm/version.rb
CHANGED
@@ -1,19 +1,19 @@
|
|
1
1
|
#!/bin/bash
|
2
2
|
# Wait for writer
|
3
3
|
echo "Waiting for MySQL-1: "
|
4
|
-
while ! (mysqladmin ping --host="127.0.0.1" --port=
|
4
|
+
while ! (mysqladmin ping --host="127.0.0.1" --port=13006 --user=root --password=password --silent 2> /dev/null); do
|
5
5
|
echo -ne "."
|
6
6
|
sleep 1
|
7
7
|
done
|
8
8
|
# Wait for reader
|
9
9
|
echo "Waiting for MySQL-2: "
|
10
|
-
while ! (mysqladmin ping --host="127.0.0.1" --port=
|
10
|
+
while ! (mysqladmin ping --host="127.0.0.1" --port=13007 --user=root --password=password --silent 2> /dev/null); do
|
11
11
|
echo -ne "."
|
12
12
|
sleep 1
|
13
13
|
done
|
14
14
|
# Wait for proxysql
|
15
15
|
echo "Waiting for ProxySQL:"
|
16
|
-
while ! (mysqladmin ping --host="127.0.0.1" --port=
|
16
|
+
while ! (mysqladmin ping --host="127.0.0.1" --port=13005 --user=root --password=password --silent 2> /dev/null); do
|
17
17
|
echo -ne "."
|
18
18
|
sleep 1
|
19
19
|
done
|
@@ -3,4 +3,4 @@ CREATE USER IF NOT EXISTS 'writer'@'%' IDENTIFIED BY 'password';
|
|
3
3
|
CREATE USER IF NOT EXISTS 'reader'@'%' IDENTIFIED BY 'password';
|
4
4
|
|
5
5
|
CREATE USER IF NOT EXISTS 'replication'@'%' IDENTIFIED BY 'password';
|
6
|
-
GRANT REPLICATION SLAVE ON *.* TO' replication'@'%'
|
6
|
+
GRANT REPLICATION SLAVE ON *.* TO' replication'@'%';
|
@@ -33,8 +33,8 @@ describe Lhm::AtomicSwitcher do
|
|
33
33
|
ar_connection = mock()
|
34
34
|
ar_connection.stubs(:data_source_exists?).returns(true)
|
35
35
|
ar_connection.stubs(:active?).returns(true)
|
36
|
-
ar_connection.stubs(:
|
37
|
-
|
36
|
+
ar_connection.stubs(:select_value).returns("dummy")
|
37
|
+
ar_connection.stubs(:execute)
|
38
38
|
.raises(ActiveRecord::StatementInvalid, 'Lock wait timeout exceeded; try restarting transaction.')
|
39
39
|
.then
|
40
40
|
.returns([["dummy"]]) # Matches initial host -> triggers retry
|
@@ -62,16 +62,12 @@ describe Lhm::AtomicSwitcher do
|
|
62
62
|
ar_connection = mock()
|
63
63
|
ar_connection.stubs(:data_source_exists?).returns(true)
|
64
64
|
ar_connection.stubs(:active?).returns(true)
|
65
|
-
ar_connection.stubs(:
|
66
|
-
|
65
|
+
ar_connection.stubs(:select_value).returns("dummy")
|
66
|
+
ar_connection.stubs(:execute)
|
67
67
|
.raises(ActiveRecord::StatementInvalid, 'Lock wait timeout exceeded; try restarting transaction.')
|
68
68
|
.then
|
69
|
-
.returns([["dummy"]]) # triggers retry 1
|
70
|
-
.then
|
71
69
|
.raises(ActiveRecord::StatementInvalid, 'Lock wait timeout exceeded; try restarting transaction.')
|
72
70
|
.then
|
73
|
-
.returns([["dummy"]]) # triggers retry 2
|
74
|
-
.then
|
75
71
|
.raises(ActiveRecord::StatementInvalid, 'Lock wait timeout exceeded; try restarting transaction.') # triggers retry 2
|
76
72
|
|
77
73
|
connection = Lhm::Connection.new(connection: ar_connection, options: {
|
@@ -74,7 +74,8 @@ describe Lhm::Chunker do
|
|
74
74
|
Lhm::Chunker.new(migration, connection, {raise_on_warnings: true, throttler: throttler, printer: printer} ).run
|
75
75
|
end
|
76
76
|
|
77
|
-
|
77
|
+
error_key = index_key("custom_primary_key_dest", "index_custom_primary_key_on_id")
|
78
|
+
assert_match "Unexpected warning found for inserted row: Duplicate entry '1001' for key '#{error_key}'", exception.message
|
78
79
|
end
|
79
80
|
|
80
81
|
it 'should copy and warn on unexpected warnings by default' do
|
@@ -87,8 +88,10 @@ describe Lhm::Chunker do
|
|
87
88
|
|
88
89
|
Lhm::Chunker.new(migration, connection, {throttler: throttler, printer: printer} ).run
|
89
90
|
|
91
|
+
error_key = index_key("custom_primary_key_dest", "index_custom_primary_key_on_id")
|
92
|
+
|
90
93
|
assert_equal 2, log_messages.length
|
91
|
-
assert log_messages[1].include?("Unexpected warning found for inserted row: Duplicate entry '1001' for key '
|
94
|
+
assert log_messages[1].include?("Unexpected warning found for inserted row: Duplicate entry '1001' for key '#{error_key}'"), log_messages
|
92
95
|
end
|
93
96
|
|
94
97
|
it 'should log two times for two unexpected warnings' do
|
@@ -103,9 +106,11 @@ describe Lhm::Chunker do
|
|
103
106
|
|
104
107
|
Lhm::Chunker.new(migration, connection, {throttler: throttler, printer: printer} ).run
|
105
108
|
|
109
|
+
error_key = index_key("custom_primary_key_dest", "index_custom_primary_key_on_id")
|
110
|
+
|
106
111
|
assert_equal 3, log_messages.length
|
107
|
-
assert log_messages[1].include?("Unexpected warning found for inserted row: Duplicate entry '1001' for key '
|
108
|
-
assert log_messages[2].include?("Unexpected warning found for inserted row: Duplicate entry '1002' for key '
|
112
|
+
assert log_messages[1].include?("Unexpected warning found for inserted row: Duplicate entry '1001' for key '#{error_key}'"), log_messages
|
113
|
+
assert log_messages[2].include?("Unexpected warning found for inserted row: Duplicate entry '1002' for key '#{error_key}'"), log_messages
|
109
114
|
end
|
110
115
|
|
111
116
|
it 'should copy and warn on unexpected warnings' do
|
@@ -118,8 +123,10 @@ describe Lhm::Chunker do
|
|
118
123
|
|
119
124
|
Lhm::Chunker.new(migration, connection, {raise_on_warnings: false, throttler: throttler, printer: printer} ).run
|
120
125
|
|
126
|
+
error_key = index_key("custom_primary_key_dest", "index_custom_primary_key_on_id")
|
127
|
+
|
121
128
|
assert_equal 2, log_messages.length
|
122
|
-
assert log_messages[1].include?("Unexpected warning found for inserted row: Duplicate entry '1001' for key '
|
129
|
+
assert log_messages[1].include?("Unexpected warning found for inserted row: Duplicate entry '1001' for key '#{error_key}'"), log_messages
|
123
130
|
end
|
124
131
|
|
125
132
|
it 'should create the modified destination, even if the source is empty' do
|
@@ -222,7 +229,7 @@ describe Lhm::Chunker do
|
|
222
229
|
def throttler.replica_connection(replica)
|
223
230
|
config = ActiveRecord::Base.connection_pool.db_config.configuration_hash.dup
|
224
231
|
config[:host] = replica
|
225
|
-
config[:port] =
|
232
|
+
config[:port] = 13007
|
226
233
|
ActiveRecord::Base.send('mysql2_connection', config)
|
227
234
|
end
|
228
235
|
end
|
@@ -261,4 +268,12 @@ describe Lhm::Chunker do
|
|
261
268
|
end
|
262
269
|
end
|
263
270
|
end
|
271
|
+
|
272
|
+
def index_key(table_name, index_name)
|
273
|
+
if mysql_version.start_with?("8")
|
274
|
+
"#{table_name}.#{index_name}"
|
275
|
+
else
|
276
|
+
index_name
|
277
|
+
end
|
278
|
+
end
|
264
279
|
end
|
@@ -2,17 +2,17 @@ master:
|
|
2
2
|
host: mysql-1
|
3
3
|
user: root
|
4
4
|
password: password
|
5
|
-
port:
|
5
|
+
port: 13006
|
6
6
|
replica:
|
7
7
|
host: mysql-2
|
8
8
|
user: root
|
9
9
|
password: password
|
10
|
-
port:
|
10
|
+
port: 13007
|
11
11
|
proxysql:
|
12
12
|
host: proxysql
|
13
13
|
user: root
|
14
14
|
password: password
|
15
|
-
port:
|
15
|
+
port: 13005
|
16
16
|
master_toxic:
|
17
17
|
host: toxiproxy
|
18
18
|
user: root
|
@@ -22,7 +22,7 @@ module IntegrationHelper
|
|
22
22
|
def self.included(base)
|
23
23
|
base.after(:each) do
|
24
24
|
cleanup_connection = new_mysql_connection
|
25
|
-
results =
|
25
|
+
results = DATABASE.query(cleanup_connection, "SELECT table_name FROM information_schema.tables WHERE table_schema = '#{$db_name}';")
|
26
26
|
table_names_for_cleanup = results.map { |row| "#{$db_name}." + row.values.first }
|
27
27
|
cleanup_connection.query("DROP TABLE IF EXISTS #{table_names_for_cleanup.join(', ')};") if table_names_for_cleanup.length > 0
|
28
28
|
end
|
@@ -35,6 +35,14 @@ module IntegrationHelper
|
|
35
35
|
@connection
|
36
36
|
end
|
37
37
|
|
38
|
+
def mysql_version
|
39
|
+
@mysql_version ||= begin
|
40
|
+
# This SQL returns a value of shape: X.Y.ZZ-AA-log
|
41
|
+
result = connection.query("SELECT VERSION()")
|
42
|
+
result.dig(0, 0).split("-", 2)[0]
|
43
|
+
end
|
44
|
+
end
|
45
|
+
|
38
46
|
def connect_proxysql!
|
39
47
|
connect!(
|
40
48
|
'127.0.0.1',
|
@@ -81,7 +89,7 @@ module IntegrationHelper
|
|
81
89
|
|
82
90
|
def ar_conn(host, port, user, password)
|
83
91
|
ActiveRecord::Base.establish_connection(
|
84
|
-
:adapter =>
|
92
|
+
:adapter => DATABASE.adapter,
|
85
93
|
:host => host,
|
86
94
|
:username => user,
|
87
95
|
:port => port,
|
@@ -171,7 +179,7 @@ module IntegrationHelper
|
|
171
179
|
end
|
172
180
|
|
173
181
|
def new_mysql_connection(role='master')
|
174
|
-
|
182
|
+
DATABASE.client.new(
|
175
183
|
host: '127.0.0.1',
|
176
184
|
database: $db_name,
|
177
185
|
username: $db_config[role]['user'],
|
@@ -9,6 +9,10 @@ describe Lhm do
|
|
9
9
|
|
10
10
|
before(:each) { connect_master!; Lhm.cleanup(true) }
|
11
11
|
|
12
|
+
let(:collation) do
|
13
|
+
mysql_version.start_with?("8.0") ? "utf8mb3_general_ci" : "utf8_general_ci"
|
14
|
+
end
|
15
|
+
|
12
16
|
describe 'id column requirement' do
|
13
17
|
it 'should migrate the table when id is pk' do
|
14
18
|
table_create(:users)
|
@@ -17,9 +21,11 @@ describe Lhm do
|
|
17
21
|
t.add_column(:logins, "int(12) default '0'")
|
18
22
|
end
|
19
23
|
|
24
|
+
expected_type = mysql_version.start_with?("8.0") ? "int" : "int(12)"
|
25
|
+
|
20
26
|
replica do
|
21
27
|
value(table_read(:users).columns['logins']).must_equal({
|
22
|
-
:type =>
|
28
|
+
:type => expected_type,
|
23
29
|
:is_nullable => 'YES',
|
24
30
|
:column_default => '0',
|
25
31
|
:comment => '',
|
@@ -35,9 +41,11 @@ describe Lhm do
|
|
35
41
|
t.add_column(:logins, "int(12) default '0'")
|
36
42
|
end
|
37
43
|
|
44
|
+
expected_type = mysql_version.start_with?("8.0") ? "int" : "int(12)"
|
45
|
+
|
38
46
|
replica do
|
39
47
|
value(table_read(:custom_primary_key).columns['logins']).must_equal({
|
40
|
-
:type =>
|
48
|
+
:type => expected_type,
|
41
49
|
:is_nullable => 'YES',
|
42
50
|
:column_default => '0',
|
43
51
|
:comment => '',
|
@@ -53,9 +61,11 @@ describe Lhm do
|
|
53
61
|
t.add_column(:logins, "int(12) default '0'")
|
54
62
|
end
|
55
63
|
|
64
|
+
expected_type = mysql_version.start_with?("8.0") ? "int" : "int(12)"
|
65
|
+
|
56
66
|
replica do
|
57
67
|
value(table_read(:composite_primary_key).columns['logins']).must_equal({
|
58
|
-
:type =>
|
68
|
+
:type => expected_type,
|
59
69
|
:is_nullable => 'YES',
|
60
70
|
:column_default => '0',
|
61
71
|
:comment => '',
|
@@ -132,9 +142,11 @@ describe Lhm do
|
|
132
142
|
t.add_column(:logins, "INT(12) DEFAULT '0'")
|
133
143
|
end
|
134
144
|
|
145
|
+
expected_type = mysql_version.start_with?("8.0") ? "int" : "int(12)"
|
146
|
+
|
135
147
|
replica do
|
136
148
|
value(table_read(:users).columns['logins']).must_equal({
|
137
|
-
:type =>
|
149
|
+
:type => expected_type,
|
138
150
|
:is_nullable => 'YES',
|
139
151
|
:column_default => '0',
|
140
152
|
:comment => '',
|
@@ -272,7 +284,7 @@ describe Lhm do
|
|
272
284
|
:is_nullable => 'NO',
|
273
285
|
:column_default => 'none',
|
274
286
|
:comment => '',
|
275
|
-
:collate =>
|
287
|
+
:collate => collation,
|
276
288
|
})
|
277
289
|
end
|
278
290
|
end
|
@@ -284,9 +296,11 @@ describe Lhm do
|
|
284
296
|
t.change_column(:id, 'int(5)')
|
285
297
|
end
|
286
298
|
|
299
|
+
expected_type = mysql_version.start_with?("8.0") ? "int" : "int(5)"
|
300
|
+
|
287
301
|
replica do
|
288
302
|
value(table_read(:small_table).columns['id']).must_equal({
|
289
|
-
:type =>
|
303
|
+
:type => expected_type,
|
290
304
|
:is_nullable => 'NO',
|
291
305
|
:column_default => nil,
|
292
306
|
:comment => '',
|
@@ -311,7 +325,7 @@ describe Lhm do
|
|
311
325
|
:is_nullable => 'YES',
|
312
326
|
:column_default => nil,
|
313
327
|
:comment => '',
|
314
|
-
:collate =>
|
328
|
+
:collate => collation,
|
315
329
|
})
|
316
330
|
|
317
331
|
result = select_one('SELECT login from users')
|
@@ -336,7 +350,7 @@ describe Lhm do
|
|
336
350
|
:is_nullable => 'YES',
|
337
351
|
:column_default => 'Superfriends',
|
338
352
|
:comment => '',
|
339
|
-
:collate =>
|
353
|
+
:collate => collation,
|
340
354
|
})
|
341
355
|
|
342
356
|
result = select_one('SELECT `fnord` from users')
|
@@ -383,11 +397,13 @@ describe Lhm do
|
|
383
397
|
t.rename_column(:reference, :ref)
|
384
398
|
end
|
385
399
|
|
400
|
+
expected_type = mysql_version.start_with?("8.0") ? "int" : "int(11)"
|
401
|
+
|
386
402
|
replica do
|
387
403
|
table_data = table_read(:users)
|
388
404
|
assert_nil table_data.columns['reference']
|
389
405
|
value(table_read(:users).columns['ref']).must_equal({
|
390
|
-
:type =>
|
406
|
+
:type => expected_type,
|
391
407
|
:is_nullable => 'YES',
|
392
408
|
:column_default => nil,
|
393
409
|
:comment => 'RefComment',
|
@@ -418,7 +434,7 @@ describe Lhm do
|
|
418
434
|
:is_nullable => 'YES',
|
419
435
|
:column_default => nil,
|
420
436
|
:comment => '',
|
421
|
-
:collate =>
|
437
|
+
:collate => collation,
|
422
438
|
})
|
423
439
|
|
424
440
|
result = select_one('SELECT `fnord` from users')
|
@@ -443,7 +459,7 @@ describe Lhm do
|
|
443
459
|
:is_nullable => 'YES',
|
444
460
|
:column_default => nil,
|
445
461
|
:comment => '',
|
446
|
-
:collate =>
|
462
|
+
:collate => collation,
|
447
463
|
})
|
448
464
|
|
449
465
|
result = select_one('SELECT `user_name` from users')
|
@@ -470,7 +486,7 @@ describe Lhm do
|
|
470
486
|
:is_nullable => 'NO',
|
471
487
|
:column_default => nil,
|
472
488
|
:comment => '',
|
473
|
-
:collate =>
|
489
|
+
:collate => collation,
|
474
490
|
})
|
475
491
|
|
476
492
|
result = select_one('SELECT `user_name` from users')
|
@@ -517,7 +533,7 @@ describe Lhm do
|
|
517
533
|
:is_nullable => 'YES',
|
518
534
|
:column_default => 'Superfriends',
|
519
535
|
:comment => '',
|
520
|
-
:collate =>
|
536
|
+
:collate => collation,
|
521
537
|
})
|
522
538
|
end
|
523
539
|
end
|
@@ -1,34 +1,34 @@
|
|
1
1
|
describe "ProxySQL integration" do
|
2
2
|
it "Should contact the writer" do
|
3
|
-
conn =
|
3
|
+
conn = DATABASE.client.new(
|
4
4
|
host: '127.0.0.1',
|
5
5
|
username: "writer",
|
6
6
|
password: "password",
|
7
|
-
port: "
|
7
|
+
port: "13005",
|
8
8
|
)
|
9
9
|
|
10
|
-
assert_equal
|
10
|
+
assert_equal DATABASE.query(conn, "SELECT @@global.hostname as host").each.first["host"], "mysql-1"
|
11
11
|
end
|
12
12
|
|
13
13
|
it "Should contact the reader" do
|
14
|
-
conn =
|
14
|
+
conn = DATABASE.client.new(
|
15
15
|
host: '127.0.0.1',
|
16
16
|
username: "reader",
|
17
17
|
password: "password",
|
18
|
-
port: "
|
18
|
+
port: "13005",
|
19
19
|
)
|
20
20
|
|
21
|
-
assert_equal
|
21
|
+
assert_equal DATABASE.query(conn, "SELECT @@global.hostname as host").each.first["host"], "mysql-2"
|
22
22
|
end
|
23
23
|
|
24
24
|
it "Should override default hostgroup from user if rule matches" do
|
25
|
-
conn =
|
25
|
+
conn = DATABASE.client.new(
|
26
26
|
host: '127.0.0.1',
|
27
27
|
username: "reader",
|
28
28
|
password: "password",
|
29
|
-
port: "
|
29
|
+
port: "13005",
|
30
30
|
)
|
31
31
|
|
32
|
-
assert_equal
|
32
|
+
assert_equal DATABASE.query(conn, "SELECT @@global.hostname as host #{Lhm::ProxySQLHelper::ANNOTATION}").each.first["host"], "mysql-1"
|
33
33
|
end
|
34
|
-
end
|
34
|
+
end
|
@@ -1,5 +1,4 @@
|
|
1
1
|
require 'yaml'
|
2
|
-
require 'mysql2'
|
3
2
|
|
4
3
|
class DBConnectionHelper
|
5
4
|
|
@@ -11,12 +10,11 @@ class DBConnectionHelper
|
|
11
10
|
end
|
12
11
|
|
13
12
|
def new_mysql_connection(role = :master, with_data = false, toxic = false)
|
14
|
-
|
15
13
|
key = role.to_s + toxic_postfix(toxic)
|
16
14
|
|
17
15
|
conn = ActiveRecord::Base.establish_connection(
|
18
16
|
:host => '127.0.0.1',
|
19
|
-
:adapter =>
|
17
|
+
:adapter => DATABASE.adapter,
|
20
18
|
:username => db_config[key]['user'],
|
21
19
|
:password => db_config[key]['password'],
|
22
20
|
:database => test_db_name,
|
@@ -49,4 +47,4 @@ class DBConnectionHelper
|
|
49
47
|
end
|
50
48
|
end
|
51
49
|
end
|
52
|
-
end
|
50
|
+
end
|
@@ -1,5 +1,4 @@
|
|
1
1
|
require 'minitest/autorun'
|
2
|
-
require 'mysql2'
|
3
2
|
require 'integration/sql_retry/lock_wait_timeout_test_helper'
|
4
3
|
require 'lhm'
|
5
4
|
|
@@ -22,7 +21,7 @@ describe Lhm::SqlRetry do
|
|
22
21
|
# Assert our pre-conditions
|
23
22
|
assert_equal 2, @helper.record_count
|
24
23
|
|
25
|
-
|
24
|
+
DATABASE.client.any_instance.stubs(:active?).returns(true)
|
26
25
|
end
|
27
26
|
|
28
27
|
after(:each) do
|
@@ -43,8 +42,8 @@ describe Lhm::SqlRetry do
|
|
43
42
|
|
44
43
|
exception = assert_raises { @helper.trigger_wait_lock }
|
45
44
|
|
46
|
-
assert_match
|
47
|
-
assert_equal
|
45
|
+
assert_match Regexp.new("Lock wait timeout exceeded; try restarting transaction"), exception.message
|
46
|
+
assert_equal DATABASE.timeout_error, exception.class
|
48
47
|
|
49
48
|
assert_equal 2, @helper.record_count # no records inserted
|
50
49
|
puts "*" * 64
|
@@ -82,10 +81,10 @@ describe Lhm::SqlRetry do
|
|
82
81
|
logs = @logger.string.split("\n")
|
83
82
|
assert_equal 2, logs.length
|
84
83
|
|
85
|
-
assert logs.first.include?("
|
84
|
+
assert logs.first.include?("Lock wait timeout exceeded; try restarting transaction' - 1 tries")
|
86
85
|
assert logs.first.include?("0.2 seconds until the next try")
|
87
86
|
|
88
|
-
assert logs.last.include?("
|
87
|
+
assert logs.last.include?("Lock wait timeout exceeded; try restarting transaction' - 2 tries")
|
89
88
|
assert logs.last.include?("0.2 seconds until the next try")
|
90
89
|
end
|
91
90
|
|
@@ -118,8 +117,8 @@ describe Lhm::SqlRetry do
|
|
118
117
|
|
119
118
|
exception = assert_raises { @helper.trigger_wait_lock }
|
120
119
|
|
121
|
-
assert_match
|
122
|
-
assert_equal
|
120
|
+
assert_match Regexp.new("Lock wait timeout exceeded; try restarting transaction"), exception.message
|
121
|
+
assert_equal DATABASE.timeout_error, exception.class
|
123
122
|
|
124
123
|
assert_equal 2, @helper.record_count # no records inserted
|
125
124
|
puts "*" * 64
|
@@ -14,13 +14,21 @@ class LockWaitTimeoutTestHelper
|
|
14
14
|
|
15
15
|
@lock_duration = lock_duration
|
16
16
|
|
17
|
-
# While implementing this, I discovered that MySQL seems to have an off-by-one
|
18
|
-
# bug with the innodb_lock_wait_timeout. If you ask it to wait 2 seconds, it will wait 3.
|
19
|
-
# In order to avoid surprisingly the user, let's account for that here, but also
|
20
|
-
# guard against a case where we go below 1, the minimum value.
|
21
|
-
raise ArgumentError, "innodb_lock_wait_timeout must be greater than or equal to 2" unless innodb_lock_wait_timeout >= 2
|
22
17
|
raise ArgumentError, "innodb_lock_wait_timeout must be an integer" if innodb_lock_wait_timeout.class != Integer
|
23
|
-
|
18
|
+
|
19
|
+
result = DATABASE.query(@main_conn, "SELECT VERSION()")
|
20
|
+
mysql_version = result.to_a.dig(0, "VERSION()").split("-", 2)[0]
|
21
|
+
|
22
|
+
if mysql_version.start_with?("8")
|
23
|
+
@innodb_lock_wait_timeout = innodb_lock_wait_timeout
|
24
|
+
else
|
25
|
+
# While implementing this, I discovered that MySQL seems to have an off-by-one
|
26
|
+
# bug with the innodb_lock_wait_timeout. If you ask it to wait 2 seconds, it will wait 3.
|
27
|
+
# In order to avoid surprisingly the user, let's account for that here, but also
|
28
|
+
# guard against a case where we go below 1, the minimum value.
|
29
|
+
raise ArgumentError, "innodb_lock_wait_timeout must be greater than or equal to 2" unless innodb_lock_wait_timeout >= 2
|
30
|
+
@innodb_lock_wait_timeout = innodb_lock_wait_timeout - 1
|
31
|
+
end
|
24
32
|
|
25
33
|
@threads = []
|
26
34
|
@queue = Queue.new
|
@@ -51,7 +59,7 @@ class LockWaitTimeoutTestHelper
|
|
51
59
|
end
|
52
60
|
|
53
61
|
def record_count(connection = main_conn)
|
54
|
-
response = connection
|
62
|
+
response = mysql_exec(connection, "SELECT COUNT(id) FROM #{test_table_name}")
|
55
63
|
response.first.values.first
|
56
64
|
end
|
57
65
|
|
@@ -79,7 +87,7 @@ class LockWaitTimeoutTestHelper
|
|
79
87
|
attr_reader :main_conn, :lock_duration, :innodb_lock_wait_timeout
|
80
88
|
|
81
89
|
def new_mysql_connection
|
82
|
-
|
90
|
+
DATABASE.client.new(
|
83
91
|
host: '127.0.0.1',
|
84
92
|
username: db_config['master']['user'],
|
85
93
|
password: db_config['master']['password'],
|
@@ -103,8 +111,8 @@ class LockWaitTimeoutTestHelper
|
|
103
111
|
private
|
104
112
|
|
105
113
|
def mysql_exec(connection, statement)
|
106
|
-
if connection.class ==
|
107
|
-
|
114
|
+
if connection.class == DATABASE.client
|
115
|
+
DATABASE.query(connection, statement)
|
108
116
|
elsif connection.class.to_s.include?("ActiveRecord")
|
109
117
|
connection.execute(statement)
|
110
118
|
else
|
@@ -2,7 +2,7 @@ class ProxySQLHelper
|
|
2
2
|
class << self
|
3
3
|
# Flips the destination hostgroup for /maintenance:lhm/ from 0 (i.e. writer) to 1 (i.e. reader)
|
4
4
|
def with_lhm_hostgroup_flip
|
5
|
-
conn =
|
5
|
+
conn = DATABASE.client.new(
|
6
6
|
host: '127.0.0.1',
|
7
7
|
username: "remote-admin",
|
8
8
|
password: "password",
|
@@ -1,5 +1,4 @@
|
|
1
1
|
require 'minitest/autorun'
|
2
|
-
require 'mysql2'
|
3
2
|
require 'lhm'
|
4
3
|
require 'toxiproxy'
|
5
4
|
|
@@ -50,7 +49,7 @@ describe Lhm::SqlRetry, "ProxiSQL tests for LHM retry" do
|
|
50
49
|
end
|
51
50
|
end
|
52
51
|
|
53
|
-
assert_equal @connection.
|
52
|
+
assert_equal 2000, @connection.select_one("SELECT * FROM #{DBConnectionHelper.test_table_name} WHERE id=2000")["id"]
|
54
53
|
|
55
54
|
logs = @logger.string.split("\n")
|
56
55
|
|
@@ -23,7 +23,7 @@ describe Lhm::Table do
|
|
23
23
|
end
|
24
24
|
|
25
25
|
it 'should parse columns' do
|
26
|
-
value(@table.columns['id'][:type]).must_match(/(bigint|int)\(\d+\)
|
26
|
+
value(@table.columns['id'][:type]).must_match(/(bigint|int)(\(\d+\))?/)
|
27
27
|
end
|
28
28
|
|
29
29
|
it 'should return true for method that should be renamed' do
|