lhm-shopify 3.4.2 → 3.5.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.github/workflows/test.yml +24 -15
- data/.gitignore +1 -6
- data/Appraisals +24 -0
- data/CHANGELOG.md +15 -0
- data/Gemfile.lock +66 -0
- data/README.md +53 -4
- data/Rakefile +10 -0
- data/dev.yml +28 -6
- data/docker-compose.yml +58 -0
- data/gemfiles/activerecord_5.2.gemfile +9 -0
- data/gemfiles/activerecord_5.2.gemfile.lock +65 -0
- data/gemfiles/activerecord_6.0.gemfile +7 -0
- data/gemfiles/activerecord_6.0.gemfile.lock +67 -0
- data/gemfiles/activerecord_6.1.gemfile +7 -0
- data/gemfiles/activerecord_6.1.gemfile.lock +66 -0
- data/gemfiles/activerecord_7.0.0.alpha2.gemfile +7 -0
- data/gemfiles/activerecord_7.0.0.alpha2.gemfile.lock +64 -0
- data/lhm.gemspec +7 -3
- data/lib/lhm/atomic_switcher.rb +4 -11
- data/lib/lhm/chunk_insert.rb +4 -10
- data/lib/lhm/chunker.rb +7 -8
- data/lib/lhm/cleanup/current.rb +3 -10
- data/lib/lhm/connection.rb +109 -0
- data/lib/lhm/entangler.rb +4 -12
- data/lib/lhm/invoker.rb +3 -3
- data/lib/lhm/proxysql_helper.rb +10 -0
- data/lib/lhm/sql_retry.rb +132 -9
- data/lib/lhm/throttler/slave_lag.rb +19 -2
- data/lib/lhm/version.rb +1 -1
- data/lib/lhm.rb +25 -9
- data/scripts/helpers/wait-for-dbs.sh +21 -0
- data/scripts/mysql/reader/create_replication.sql +10 -0
- data/scripts/mysql/writer/create_test_db.sql +1 -0
- data/scripts/mysql/writer/create_users.sql +6 -0
- data/scripts/proxysql/proxysql.cnf +117 -0
- data/spec/integration/atomic_switcher_spec.rb +38 -13
- data/spec/integration/chunk_insert_spec.rb +2 -1
- data/spec/integration/chunker_spec.rb +1 -1
- data/spec/integration/database.yml +25 -0
- data/spec/integration/entangler_spec.rb +3 -1
- data/spec/integration/integration_helper.rb +24 -9
- data/spec/integration/lhm_spec.rb +75 -0
- data/spec/integration/proxysql_spec.rb +34 -0
- data/spec/integration/sql_retry/db_connection_helper.rb +52 -0
- data/spec/integration/sql_retry/lock_wait_spec.rb +8 -6
- data/spec/integration/sql_retry/lock_wait_timeout_test_helper.rb +17 -4
- data/spec/integration/sql_retry/proxysql_helper.rb +22 -0
- data/spec/integration/sql_retry/retry_with_proxysql_spec.rb +108 -0
- data/spec/integration/toxiproxy_helper.rb +40 -0
- data/spec/test_helper.rb +21 -0
- data/spec/unit/chunk_insert_spec.rb +7 -2
- data/spec/unit/chunker_spec.rb +46 -42
- data/spec/unit/connection_spec.rb +86 -0
- data/spec/unit/entangler_spec.rb +51 -10
- data/spec/unit/lhm_spec.rb +17 -0
- data/spec/unit/throttler/slave_lag_spec.rb +13 -8
- metadata +85 -14
- data/bin/.gitkeep +0 -0
- data/dbdeployer/config.json +0 -32
- data/dbdeployer/install.sh +0 -64
- data/gemfiles/ar-2.3_mysql.gemfile +0 -6
- data/gemfiles/ar-3.2_mysql.gemfile +0 -5
- data/gemfiles/ar-3.2_mysql2.gemfile +0 -5
- data/gemfiles/ar-4.0_mysql2.gemfile +0 -5
- data/gemfiles/ar-4.1_mysql2.gemfile +0 -5
- data/gemfiles/ar-4.2_mysql2.gemfile +0 -5
- data/gemfiles/ar-5.0_mysql2.gemfile +0 -5
@@ -0,0 +1,6 @@
|
|
1
|
+
# Creates replication user in Writer
|
2
|
+
CREATE USER IF NOT EXISTS 'writer'@'%' IDENTIFIED BY 'password';
|
3
|
+
CREATE USER IF NOT EXISTS 'reader'@'%' IDENTIFIED BY 'password';
|
4
|
+
|
5
|
+
CREATE USER IF NOT EXISTS 'replication'@'%' IDENTIFIED BY 'password';
|
6
|
+
GRANT REPLICATION SLAVE ON *.* TO' replication'@'%' IDENTIFIED BY 'password';
|
@@ -0,0 +1,117 @@
|
|
1
|
+
#file proxysql.cfg
|
2
|
+
|
3
|
+
datadir="/var/lib/proxysql"
|
4
|
+
restart_on_missing_heartbeats=999999
|
5
|
+
query_parser_token_delimiters=","
|
6
|
+
query_parser_key_value_delimiters=":"
|
7
|
+
unit_of_work_identifiers="consistent_read_id"
|
8
|
+
|
9
|
+
admin_variables=
|
10
|
+
{
|
11
|
+
mysql_ifaces="0.0.0.0:6032"
|
12
|
+
admin_credentials="admin:password;remote-admin:password"
|
13
|
+
}
|
14
|
+
|
15
|
+
mysql_servers =
|
16
|
+
(
|
17
|
+
{
|
18
|
+
address="mysql-1"
|
19
|
+
port=3306
|
20
|
+
hostgroup=0
|
21
|
+
max_connections=200
|
22
|
+
},
|
23
|
+
{
|
24
|
+
address="mysql-2"
|
25
|
+
port=3306
|
26
|
+
hostgroup=1
|
27
|
+
max_connections=200
|
28
|
+
}
|
29
|
+
)
|
30
|
+
|
31
|
+
mysql_variables=
|
32
|
+
{
|
33
|
+
session_idle_ms=1
|
34
|
+
auto_increment_delay_multiplex=0
|
35
|
+
|
36
|
+
threads=8
|
37
|
+
max_connections=100000
|
38
|
+
interfaces="0.0.0.0:3306"
|
39
|
+
server_version="5.7.18-proxysql"
|
40
|
+
connect_timeout_server=10000
|
41
|
+
connect_timeout_server_max=10000
|
42
|
+
connect_retries_on_failure=0
|
43
|
+
default_charset="utf8mb4"
|
44
|
+
free_connections_pct=100
|
45
|
+
connection_warming=true
|
46
|
+
max_allowed_packet=16777216
|
47
|
+
monitor_enabled=false
|
48
|
+
query_retries_on_failure=0
|
49
|
+
shun_on_failures=999999
|
50
|
+
shun_recovery_time_sec=0
|
51
|
+
kill_backend_connection_when_disconnect=false
|
52
|
+
stats_time_backend_query=false
|
53
|
+
stats_time_query_processor=false
|
54
|
+
max_stmts_per_connection=5
|
55
|
+
default_max_latency_ms=999999
|
56
|
+
wait_timeout=1800000
|
57
|
+
eventslog_format=3
|
58
|
+
log_multiplexing_disabled=true
|
59
|
+
log_unhealthy_connections=false
|
60
|
+
}
|
61
|
+
|
62
|
+
# defines all the MySQL users
|
63
|
+
mysql_users:
|
64
|
+
(
|
65
|
+
{
|
66
|
+
username = "root"
|
67
|
+
password = "password"
|
68
|
+
default_hostgroup = 0
|
69
|
+
max_connections=1000
|
70
|
+
active = 1
|
71
|
+
},
|
72
|
+
{
|
73
|
+
username = "writer"
|
74
|
+
password = "password"
|
75
|
+
default_hostgroup = 0
|
76
|
+
max_connections=50000
|
77
|
+
active = 1
|
78
|
+
transaction_persistent=1
|
79
|
+
},
|
80
|
+
{
|
81
|
+
username = "reader"
|
82
|
+
password = "password"
|
83
|
+
default_hostgroup = 1
|
84
|
+
max_connections=50000
|
85
|
+
active = 1
|
86
|
+
transaction_persistent=1
|
87
|
+
}
|
88
|
+
)
|
89
|
+
|
90
|
+
#defines MySQL Query Rules
|
91
|
+
mysql_query_rules:
|
92
|
+
(
|
93
|
+
{
|
94
|
+
rule_id = 1
|
95
|
+
active = 1
|
96
|
+
match_digest = "@@SESSION"
|
97
|
+
multiplex = 2
|
98
|
+
},
|
99
|
+
{
|
100
|
+
rule_id = 2
|
101
|
+
active = 1
|
102
|
+
match_digest = "@@global\.server_id"
|
103
|
+
multiplex = 2
|
104
|
+
},
|
105
|
+
{
|
106
|
+
rule_id = 3
|
107
|
+
active = 1
|
108
|
+
match_digest = "@@global\.hostname"
|
109
|
+
multiplex = 2
|
110
|
+
},
|
111
|
+
{
|
112
|
+
rule_id = 4
|
113
|
+
active = 1
|
114
|
+
match_pattern = "maintenance:lhm"
|
115
|
+
destination_hostgroup = 0
|
116
|
+
}
|
117
|
+
)
|
@@ -6,6 +6,7 @@ require File.expand_path(File.dirname(__FILE__)) + '/integration_helper'
|
|
6
6
|
require 'lhm/table'
|
7
7
|
require 'lhm/migration'
|
8
8
|
require 'lhm/atomic_switcher'
|
9
|
+
require 'lhm/connection'
|
9
10
|
|
10
11
|
describe Lhm::AtomicSwitcher do
|
11
12
|
include IntegrationHelper
|
@@ -15,9 +16,9 @@ describe Lhm::AtomicSwitcher do
|
|
15
16
|
describe 'switching' do
|
16
17
|
before(:each) do
|
17
18
|
Thread.abort_on_exception = true
|
18
|
-
@origin
|
19
|
+
@origin = table_create('origin')
|
19
20
|
@destination = table_create('destination')
|
20
|
-
@migration
|
21
|
+
@migration = Lhm::Migration.new(@origin, @destination)
|
21
22
|
@logs = StringIO.new
|
22
23
|
Lhm.logger = Logger.new(@logs)
|
23
24
|
@connection.execute('SET GLOBAL innodb_lock_wait_timeout=3')
|
@@ -29,26 +30,48 @@ describe Lhm::AtomicSwitcher do
|
|
29
30
|
end
|
30
31
|
|
31
32
|
it 'should retry and log on lock wait timeouts' do
|
32
|
-
|
33
|
-
|
34
|
-
|
33
|
+
ar_connection = mock()
|
34
|
+
ar_connection.stubs(:data_source_exists?).returns(true)
|
35
|
+
ar_connection.stubs(:active?).returns(true)
|
36
|
+
ar_connection.stubs(:execute).returns([["dummy"]], [["dummy"]], [["dummy"]])
|
37
|
+
.then
|
38
|
+
.raises(ActiveRecord::StatementInvalid, 'Lock wait timeout exceeded; try restarting transaction.')
|
39
|
+
.then
|
40
|
+
.returns([["dummy"]]) # Matches initial host -> triggers retry
|
35
41
|
|
36
|
-
|
42
|
+
connection = Lhm::Connection.new(connection: ar_connection, options: {reconnect_with_consistent_host: true})
|
43
|
+
|
44
|
+
|
45
|
+
switcher = Lhm::AtomicSwitcher.new(@migration, connection, retriable: { tries: 3, base_interval: 0 })
|
37
46
|
|
38
47
|
assert switcher.run
|
39
48
|
|
40
49
|
log_messages = @logs.string.split("\n")
|
41
50
|
assert_equal(2, log_messages.length)
|
42
51
|
assert log_messages[0].include? "Starting run of class=Lhm::AtomicSwitcher"
|
52
|
+
# On failure of this assertion, check for Lhm::Connection#file
|
43
53
|
assert log_messages[1].include? "[AtomicSwitcher] ActiveRecord::StatementInvalid: 'Lock wait timeout exceeded; try restarting transaction.' - 1 tries"
|
44
54
|
end
|
45
55
|
|
46
56
|
it 'should give up on lock wait timeouts after a configured number of tries' do
|
47
|
-
|
48
|
-
|
49
|
-
|
50
|
-
|
51
|
-
|
57
|
+
ar_connection = mock()
|
58
|
+
ar_connection.stubs(:data_source_exists?).returns(true)
|
59
|
+
ar_connection.stubs(:active?).returns(true)
|
60
|
+
ar_connection.stubs(:execute).returns([["dummy"]], [["dummy"]], [["dummy"]])
|
61
|
+
.then
|
62
|
+
.raises(ActiveRecord::StatementInvalid, 'Lock wait timeout exceeded; try restarting transaction.')
|
63
|
+
.then
|
64
|
+
.returns([["dummy"]]) # triggers retry 1
|
65
|
+
.then
|
66
|
+
.raises(ActiveRecord::StatementInvalid, 'Lock wait timeout exceeded; try restarting transaction.')
|
67
|
+
.then
|
68
|
+
.returns([["dummy"]]) # triggers retry 2
|
69
|
+
.then
|
70
|
+
.raises(ActiveRecord::StatementInvalid, 'Lock wait timeout exceeded; try restarting transaction.') # triggers retry 2
|
71
|
+
|
72
|
+
connection = Lhm::Connection.new(connection: ar_connection, options: {reconnect_with_consistent_host: true})
|
73
|
+
|
74
|
+
switcher = Lhm::AtomicSwitcher.new(@migration, connection, retriable: { tries: 2, base_interval: 0 })
|
52
75
|
|
53
76
|
assert_raises(ActiveRecord::StatementInvalid) { switcher.run }
|
54
77
|
end
|
@@ -62,8 +85,10 @@ describe Lhm::AtomicSwitcher do
|
|
62
85
|
end
|
63
86
|
|
64
87
|
it "should raise when destination doesn't exist" do
|
65
|
-
|
66
|
-
|
88
|
+
ar_connection = mock()
|
89
|
+
ar_connection.stubs(:data_source_exists?).returns(false)
|
90
|
+
|
91
|
+
connection = Lhm::Connection.new(connection: ar_connection)
|
67
92
|
|
68
93
|
switcher = Lhm::AtomicSwitcher.new(@migration, connection)
|
69
94
|
|
@@ -11,7 +11,8 @@ describe Lhm::ChunkInsert do
|
|
11
11
|
@destination = table_create(:destination)
|
12
12
|
@migration = Lhm::Migration.new(@origin, @destination)
|
13
13
|
execute("insert into origin set id = 1001")
|
14
|
-
@
|
14
|
+
@connection = Lhm::Connection.new(connection: connection)
|
15
|
+
@instance = Lhm::ChunkInsert.new(@migration, @connection, 1001, 1001)
|
15
16
|
end
|
16
17
|
|
17
18
|
it "returns the count" do
|
@@ -223,7 +223,7 @@ describe Lhm::Chunker do
|
|
223
223
|
|
224
224
|
if master_slave_mode?
|
225
225
|
def throttler.slave_connection(slave)
|
226
|
-
config = ActiveRecord::Base.connection_pool.
|
226
|
+
config = ActiveRecord::Base.connection_pool.db_config.configuration_hash.dup
|
227
227
|
config[:host] = slave
|
228
228
|
config[:port] = 3307
|
229
229
|
ActiveRecord::Base.send('mysql2_connection', config)
|
@@ -0,0 +1,25 @@
|
|
1
|
+
master:
|
2
|
+
host: mysql-1
|
3
|
+
user: root
|
4
|
+
password: password
|
5
|
+
port: 33006
|
6
|
+
slave:
|
7
|
+
host: mysql-2
|
8
|
+
user: root
|
9
|
+
password: password
|
10
|
+
port: 33007
|
11
|
+
proxysql:
|
12
|
+
host: proxysql
|
13
|
+
user: root
|
14
|
+
password: password
|
15
|
+
port: 33005
|
16
|
+
master_toxic:
|
17
|
+
host: toxiproxy
|
18
|
+
user: root
|
19
|
+
password: password
|
20
|
+
port: 22220
|
21
|
+
proxysql_toxic:
|
22
|
+
host: toxiproxy
|
23
|
+
user: root
|
24
|
+
password: password
|
25
|
+
port: 22222
|
@@ -6,6 +6,7 @@ require File.expand_path(File.dirname(__FILE__)) + '/integration_helper'
|
|
6
6
|
require 'lhm/table'
|
7
7
|
require 'lhm/migration'
|
8
8
|
require 'lhm/entangler'
|
9
|
+
require 'lhm/connection'
|
9
10
|
|
10
11
|
describe Lhm::Entangler do
|
11
12
|
include IntegrationHelper
|
@@ -17,7 +18,8 @@ describe Lhm::Entangler do
|
|
17
18
|
@origin = table_create('origin')
|
18
19
|
@destination = table_create('destination')
|
19
20
|
@migration = Lhm::Migration.new(@origin, @destination)
|
20
|
-
@
|
21
|
+
@connection = Lhm::Connection.new(connection: connection)
|
22
|
+
@entangler = Lhm::Entangler.new(@migration, @connection)
|
21
23
|
end
|
22
24
|
|
23
25
|
it 'should replay inserts from origin into destination' do
|
@@ -35,13 +35,21 @@ module IntegrationHelper
|
|
35
35
|
@connection
|
36
36
|
end
|
37
37
|
|
38
|
+
def connect_proxysql!
|
39
|
+
connect!(
|
40
|
+
'127.0.0.1',
|
41
|
+
$db_config['proxysql']['port'],
|
42
|
+
$db_config['proxysql']['user'],
|
43
|
+
$db_config['proxysql']['password'],
|
44
|
+
)
|
45
|
+
end
|
46
|
+
|
38
47
|
def connect_master!
|
39
48
|
connect!(
|
40
49
|
'127.0.0.1',
|
41
50
|
$db_config['master']['port'],
|
42
51
|
$db_config['master']['user'],
|
43
52
|
$db_config['master']['password'],
|
44
|
-
$db_config['master']['socket']
|
45
53
|
)
|
46
54
|
end
|
47
55
|
|
@@ -51,28 +59,35 @@ module IntegrationHelper
|
|
51
59
|
$db_config['slave']['port'],
|
52
60
|
$db_config['slave']['user'],
|
53
61
|
$db_config['slave']['password'],
|
54
|
-
$db_config['slave']['socket']
|
55
62
|
)
|
56
63
|
end
|
57
64
|
|
58
|
-
def
|
59
|
-
|
60
|
-
|
65
|
+
def connect_master_with_toxiproxy!(with_retry: false)
|
66
|
+
connect!(
|
67
|
+
'127.0.0.1',
|
68
|
+
$db_config['master_toxic']['port'],
|
69
|
+
$db_config['master_toxic']['user'],
|
70
|
+
$db_config['master_toxic']['password'],
|
71
|
+
with_retry)
|
72
|
+
end
|
73
|
+
|
74
|
+
def connect!(hostname, port, user, password, with_retry = false)
|
75
|
+
adapter = ar_conn(hostname, port, user, password)
|
76
|
+
Lhm.setup(adapter,{reconnect_with_consistent_host: with_retry})
|
61
77
|
unless defined?(@@cleaned_up)
|
62
78
|
Lhm.cleanup(true)
|
63
79
|
@@cleaned_up = true
|
64
80
|
end
|
65
|
-
@connection =
|
81
|
+
@connection = Lhm.connection
|
66
82
|
end
|
67
83
|
|
68
|
-
def ar_conn(host, port, user, password
|
84
|
+
def ar_conn(host, port, user, password)
|
69
85
|
ActiveRecord::Base.establish_connection(
|
70
86
|
:adapter => 'mysql2',
|
71
87
|
:host => host,
|
72
88
|
:username => user,
|
73
89
|
:port => port,
|
74
90
|
:password => password,
|
75
|
-
:socket => socket,
|
76
91
|
:database => $db_name
|
77
92
|
)
|
78
93
|
ActiveRecord::Base.connection
|
@@ -122,7 +137,7 @@ module IntegrationHelper
|
|
122
137
|
# Helps testing behaviour when another client locks the db
|
123
138
|
def start_locking_thread(lock_for, queue, locking_query)
|
124
139
|
Thread.new do
|
125
|
-
conn =
|
140
|
+
conn = new_mysql_connection
|
126
141
|
conn.query('BEGIN')
|
127
142
|
conn.query(locking_query)
|
128
143
|
queue.push(true)
|
@@ -2,6 +2,7 @@
|
|
2
2
|
# Schmidt
|
3
3
|
|
4
4
|
require File.expand_path(File.dirname(__FILE__)) + '/integration_helper'
|
5
|
+
require 'integration/toxiproxy_helper'
|
5
6
|
|
6
7
|
describe Lhm do
|
7
8
|
include IntegrationHelper
|
@@ -581,5 +582,79 @@ describe Lhm do
|
|
581
582
|
end
|
582
583
|
end
|
583
584
|
end
|
585
|
+
|
586
|
+
describe 'connection' do
|
587
|
+
include ToxiproxyHelper
|
588
|
+
|
589
|
+
before(:each) do
|
590
|
+
@logs = StringIO.new
|
591
|
+
Lhm.logger = Logger.new(@logs)
|
592
|
+
end
|
593
|
+
|
594
|
+
it " should not try to reconnect if reconnect_with_consistent_host is not provided" do
|
595
|
+
connect_master_with_toxiproxy!(with_retry: false)
|
596
|
+
|
597
|
+
table_create(:users)
|
598
|
+
100.times { |n| execute("insert into users set reference = '#{ n }'") }
|
599
|
+
|
600
|
+
assert_raises ActiveRecord::StatementInvalid do
|
601
|
+
Toxiproxy[:mysql_master].down do
|
602
|
+
Lhm.change_table(:users, :atomic_switch => false) do |t|
|
603
|
+
t.ddl("ALTER TABLE #{t.name} CHANGE id id bigint (20) NOT NULL")
|
604
|
+
t.ddl("ALTER TABLE #{t.name} DROP PRIMARY KEY, ADD PRIMARY KEY (username, id)")
|
605
|
+
t.ddl("ALTER TABLE #{t.name} ADD INDEX (id)")
|
606
|
+
t.ddl("ALTER TABLE #{t.name} CHANGE id id bigint (20) NOT NULL AUTO_INCREMENT")
|
607
|
+
end
|
608
|
+
end
|
609
|
+
end
|
610
|
+
end
|
611
|
+
|
612
|
+
it "should reconnect if reconnect_with_consistent_host is true" do
|
613
|
+
connect_master_with_toxiproxy!(with_retry: true)
|
614
|
+
mysql_disabled = false
|
615
|
+
|
616
|
+
table_create(:users)
|
617
|
+
100.times { |n| execute("insert into users set reference = '#{ n }'") }
|
618
|
+
|
619
|
+
# Redeclare Lhm::ChunkInsert to use Hook to disable MySQL writer for 3 seconds before first insert
|
620
|
+
Lhm::ChunkInsert.class_eval do
|
621
|
+
extend AfterDo
|
622
|
+
|
623
|
+
before(:insert_and_return_count_of_rows_created) do
|
624
|
+
unless mysql_disabled
|
625
|
+
mysql_disabled = true
|
626
|
+
Thread.new do
|
627
|
+
Toxiproxy[:mysql_master].down do
|
628
|
+
sleep 3
|
629
|
+
end
|
630
|
+
end
|
631
|
+
end
|
632
|
+
end
|
633
|
+
|
634
|
+
# Need to call `#method_added` manually to have the hooks take into effect
|
635
|
+
method_added(:insert_and_return_count_of_rows_created)
|
636
|
+
end
|
637
|
+
|
638
|
+
Lhm.change_table(:users, :atomic_switch => false) do |t|
|
639
|
+
t.ddl("ALTER TABLE #{t.name} CHANGE id id bigint (20) NOT NULL")
|
640
|
+
t.ddl("ALTER TABLE #{t.name} DROP PRIMARY KEY, ADD PRIMARY KEY (username, id)")
|
641
|
+
t.ddl("ALTER TABLE #{t.name} ADD INDEX (id)")
|
642
|
+
t.ddl("ALTER TABLE #{t.name} CHANGE id id bigint (20) NOT NULL AUTO_INCREMENT")
|
643
|
+
end
|
644
|
+
|
645
|
+
log_lines = @logs.string.split("\n")
|
646
|
+
|
647
|
+
assert log_lines.one?{ |line| line.include?("Lost connection to MySQL, will retry to connect to same host")}
|
648
|
+
assert log_lines.any?{ |line| line.include?("Lost connection to MySQL server at 'reading initial communication packet'")}
|
649
|
+
assert log_lines.one?{ |line| line.include?("LHM successfully reconnected to initial host")}
|
650
|
+
assert log_lines.one?{ |line| line.include?("100% complete")}
|
651
|
+
|
652
|
+
Lhm::ChunkInsert.remove_all_callbacks
|
653
|
+
|
654
|
+
slave do
|
655
|
+
value(count_all(:users)).must_equal(100)
|
656
|
+
end
|
657
|
+
end
|
658
|
+
end
|
584
659
|
end
|
585
660
|
end
|
@@ -0,0 +1,34 @@
|
|
1
|
+
describe "ProxySQL integration" do
|
2
|
+
it "Should contact the writer" do
|
3
|
+
conn = Mysql2::Client.new(
|
4
|
+
host: '127.0.0.1',
|
5
|
+
username: "writer",
|
6
|
+
password: "password",
|
7
|
+
port: "33005",
|
8
|
+
)
|
9
|
+
|
10
|
+
assert_equal conn.query("SELECT @@global.hostname as host").each.first["host"], "mysql-1"
|
11
|
+
end
|
12
|
+
|
13
|
+
it "Should contact the reader" do
|
14
|
+
conn = Mysql2::Client.new(
|
15
|
+
host: '127.0.0.1',
|
16
|
+
username: "reader",
|
17
|
+
password: "password",
|
18
|
+
port: "33005",
|
19
|
+
)
|
20
|
+
|
21
|
+
assert_equal conn.query("SELECT @@global.hostname as host").each.first["host"], "mysql-2"
|
22
|
+
end
|
23
|
+
|
24
|
+
it "Should override default hostgroup from user if rule matches" do
|
25
|
+
conn = Mysql2::Client.new(
|
26
|
+
host: '127.0.0.1',
|
27
|
+
username: "reader",
|
28
|
+
password: "password",
|
29
|
+
port: "33005",
|
30
|
+
)
|
31
|
+
|
32
|
+
assert_equal conn.query("SELECT @@global.hostname as host #{Lhm::ProxySQLHelper::ANNOTATION}").each.first["host"], "mysql-1"
|
33
|
+
end
|
34
|
+
end
|
@@ -0,0 +1,52 @@
|
|
1
|
+
require 'yaml'
|
2
|
+
require 'mysql2'
|
3
|
+
|
4
|
+
class DBConnectionHelper
|
5
|
+
|
6
|
+
DATABASE_CONFIG_FILE = "database.yml"
|
7
|
+
|
8
|
+
class << self
|
9
|
+
def db_config
|
10
|
+
@db_config ||= YAML.load_file(File.expand_path(File.dirname(__FILE__)) + "/../#{DATABASE_CONFIG_FILE}")
|
11
|
+
end
|
12
|
+
|
13
|
+
def new_mysql_connection(role = :master, with_data = false, toxic = false)
|
14
|
+
|
15
|
+
key = role.to_s + toxic_postfix(toxic)
|
16
|
+
|
17
|
+
conn = ActiveRecord::Base.establish_connection(
|
18
|
+
:host => '127.0.0.1',
|
19
|
+
:adapter => "mysql2",
|
20
|
+
:username => db_config[key]['user'],
|
21
|
+
:password => db_config[key]['password'],
|
22
|
+
:database => test_db_name,
|
23
|
+
:port => db_config[key]['port']
|
24
|
+
)
|
25
|
+
conn = conn.connection
|
26
|
+
init_with_dummy_data(conn) if with_data
|
27
|
+
conn
|
28
|
+
end
|
29
|
+
|
30
|
+
def toxic_postfix(toxic)
|
31
|
+
toxic ? "_toxic" : ""
|
32
|
+
end
|
33
|
+
|
34
|
+
def test_db_name
|
35
|
+
@test_db_name ||= "test"
|
36
|
+
end
|
37
|
+
|
38
|
+
def test_table_name
|
39
|
+
@test_table_name ||= "test"
|
40
|
+
end
|
41
|
+
|
42
|
+
def init_with_dummy_data(conn)
|
43
|
+
conn.execute("DROP TABLE IF EXISTS #{test_table_name} ")
|
44
|
+
conn.execute("CREATE TABLE #{test_table_name} (id int)")
|
45
|
+
|
46
|
+
1.upto(9) do |i|
|
47
|
+
query = "INSERT INTO #{test_table_name} (id) VALUE (#{i})"
|
48
|
+
conn.execute(query)
|
49
|
+
end
|
50
|
+
end
|
51
|
+
end
|
52
|
+
end
|
@@ -21,6 +21,8 @@ describe Lhm::SqlRetry do
|
|
21
21
|
|
22
22
|
# Assert our pre-conditions
|
23
23
|
assert_equal 2, @helper.record_count
|
24
|
+
|
25
|
+
Mysql2::Client.any_instance.stubs(:active?).returns(true)
|
24
26
|
end
|
25
27
|
|
26
28
|
after(:each) do
|
@@ -41,7 +43,7 @@ describe Lhm::SqlRetry do
|
|
41
43
|
|
42
44
|
exception = assert_raises { @helper.trigger_wait_lock }
|
43
45
|
|
44
|
-
|
46
|
+
assert_match /Lock wait timeout exceeded; try restarting transaction/, exception.message
|
45
47
|
assert_equal Mysql2::Error::TimeoutError, exception.class
|
46
48
|
|
47
49
|
assert_equal 2, @helper.record_count # no records inserted
|
@@ -53,12 +55,12 @@ describe Lhm::SqlRetry do
|
|
53
55
|
it "successfully executes the SQL despite the errors encountered" do
|
54
56
|
# Start a thread to retry, once the lock is held, execute the block
|
55
57
|
@helper.with_waiting_lock do |waiting_connection|
|
56
|
-
sql_retry = Lhm::SqlRetry.new(waiting_connection, {
|
58
|
+
sql_retry = Lhm::SqlRetry.new(waiting_connection, retry_options: {
|
57
59
|
base_interval: 0.2, # first retry after 200ms
|
58
60
|
multiplier: 1, # subsequent retries wait 1x longer than first retry (no change)
|
59
61
|
tries: 3, # we only need 3 tries (including the first) for the scenario described below
|
60
62
|
rand_factor: 0 # do not introduce randomness to wait timer
|
61
|
-
})
|
63
|
+
}, reconnect_with_consistent_host: false)
|
62
64
|
|
63
65
|
# RetryTestHelper is configured to hold lock for 5 seconds and timeout after 2 seconds.
|
64
66
|
# Therefore the sequence of events will be:
|
@@ -96,12 +98,12 @@ describe Lhm::SqlRetry do
|
|
96
98
|
puts "*" * 64
|
97
99
|
# Start a thread to retry, once the lock is held, execute the block
|
98
100
|
@helper.with_waiting_lock do |waiting_connection|
|
99
|
-
sql_retry = Lhm::SqlRetry.new(waiting_connection, {
|
101
|
+
sql_retry = Lhm::SqlRetry.new(waiting_connection, retry_options: {
|
100
102
|
base_interval: 0.2, # first retry after 200ms
|
101
103
|
multiplier: 1, # subsequent retries wait 1x longer than first retry (no change)
|
102
104
|
tries: 2, # we need 3 tries (including the first) for the scenario described below, but we only get two...we will fail
|
103
105
|
rand_factor: 0 # do not introduce randomness to wait timer
|
104
|
-
})
|
106
|
+
}, reconnect_with_consistent_host: false)
|
105
107
|
|
106
108
|
# RetryTestHelper is configured to hold lock for 5 seconds and timeout after 2 seconds.
|
107
109
|
# Therefore the sequence of events will be:
|
@@ -116,7 +118,7 @@ describe Lhm::SqlRetry do
|
|
116
118
|
|
117
119
|
exception = assert_raises { @helper.trigger_wait_lock }
|
118
120
|
|
119
|
-
|
121
|
+
assert_match /Lock wait timeout exceeded; try restarting transaction/, exception.message
|
120
122
|
assert_equal Mysql2::Error::TimeoutError, exception.class
|
121
123
|
|
122
124
|
assert_equal 2, @helper.record_count # no records inserted
|
@@ -1,5 +1,7 @@
|
|
1
|
-
require '
|
1
|
+
require 'integration/integration_helper'
|
2
|
+
|
2
3
|
class LockWaitTimeoutTestHelper
|
4
|
+
|
3
5
|
def initialize(lock_duration:, innodb_lock_wait_timeout:)
|
4
6
|
# This connection will be used exclusively to setup the test,
|
5
7
|
# assert pre-conditions and assert post-conditions.
|
@@ -68,7 +70,7 @@ class LockWaitTimeoutTestHelper
|
|
68
70
|
|
69
71
|
def insert_records_at_ids(connection, ids)
|
70
72
|
ids.each do |id|
|
71
|
-
connection
|
73
|
+
mysql_exec(connection, "INSERT INTO #{test_table_name} (id) VALUES (#{id})")
|
72
74
|
end
|
73
75
|
end
|
74
76
|
|
@@ -79,11 +81,10 @@ class LockWaitTimeoutTestHelper
|
|
79
81
|
def new_mysql_connection
|
80
82
|
Mysql2::Client.new(
|
81
83
|
host: '127.0.0.1',
|
82
|
-
database: test_db_name,
|
83
84
|
username: db_config['master']['user'],
|
84
85
|
password: db_config['master']['password'],
|
85
86
|
port: db_config['master']['port'],
|
86
|
-
|
87
|
+
database: test_db_name
|
87
88
|
)
|
88
89
|
end
|
89
90
|
|
@@ -98,4 +99,16 @@ class LockWaitTimeoutTestHelper
|
|
98
99
|
def test_table_name
|
99
100
|
@test_table_name ||= "lock_wait"
|
100
101
|
end
|
102
|
+
|
103
|
+
private
|
104
|
+
|
105
|
+
def mysql_exec(connection, statement)
|
106
|
+
if connection.class == Mysql2::Client
|
107
|
+
connection.query(statement)
|
108
|
+
elsif connection.class.to_s.include?("ActiveRecord")
|
109
|
+
connection.execute(statement)
|
110
|
+
else
|
111
|
+
raise StandardError.new("Unrecognized MySQL client")
|
112
|
+
end
|
113
|
+
end
|
101
114
|
end
|
@@ -0,0 +1,22 @@
|
|
1
|
+
class ProxySQLHelper
|
2
|
+
class << self
|
3
|
+
# Flips the destination hostgroup for /maintenance:lhm/ from 0 (i.e. writer) to 1 (i.e. reader)
|
4
|
+
def with_lhm_hostgroup_flip
|
5
|
+
conn = Mysql2::Client.new(
|
6
|
+
host: '127.0.0.1',
|
7
|
+
username: "remote-admin",
|
8
|
+
password: "password",
|
9
|
+
port: "6032",
|
10
|
+
)
|
11
|
+
|
12
|
+
begin
|
13
|
+
conn.query("UPDATE mysql_query_rules SET destination_hostgroup=1 WHERE match_pattern=\"maintenance:lhm\"")
|
14
|
+
conn.query("LOAD MYSQL QUERY RULES TO RUNTIME;")
|
15
|
+
yield
|
16
|
+
ensure
|
17
|
+
conn.query("UPDATE mysql_query_rules SET destination_hostgroup=0 WHERE match_pattern=\"maintenance:lhm\"")
|
18
|
+
conn.query("LOAD MYSQL QUERY RULES TO RUNTIME;")
|
19
|
+
end
|
20
|
+
end
|
21
|
+
end
|
22
|
+
end
|