lhm-shopify 3.4.1 → 3.5.2
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.github/workflows/test.yml +24 -15
- data/.gitignore +1 -6
- data/Appraisals +24 -0
- data/CHANGELOG.md +15 -0
- data/Gemfile.lock +66 -0
- data/README.md +45 -5
- data/Rakefile +10 -0
- data/dev.yml +28 -6
- data/docker-compose.yml +58 -0
- data/gemfiles/activerecord_5.2.gemfile +9 -0
- data/gemfiles/activerecord_5.2.gemfile.lock +65 -0
- data/gemfiles/activerecord_6.0.gemfile +7 -0
- data/gemfiles/activerecord_6.0.gemfile.lock +67 -0
- data/gemfiles/activerecord_6.1.gemfile +7 -0
- data/gemfiles/activerecord_6.1.gemfile.lock +66 -0
- data/gemfiles/activerecord_7.0.0.alpha2.gemfile +7 -0
- data/gemfiles/activerecord_7.0.0.alpha2.gemfile.lock +64 -0
- data/lhm.gemspec +7 -3
- data/lib/lhm/atomic_switcher.rb +4 -11
- data/lib/lhm/chunk_insert.rb +4 -10
- data/lib/lhm/chunker.rb +15 -19
- data/lib/lhm/cleanup/current.rb +3 -10
- data/lib/lhm/connection.rb +109 -0
- data/lib/lhm/entangler.rb +4 -12
- data/lib/lhm/invoker.rb +3 -3
- data/lib/lhm/proxysql_helper.rb +10 -0
- data/lib/lhm/sql_retry.rb +132 -9
- data/lib/lhm/throttler/slave_lag.rb +19 -2
- data/lib/lhm/version.rb +1 -1
- data/lib/lhm.rb +25 -9
- data/scripts/helpers/wait-for-dbs.sh +21 -0
- data/scripts/mysql/reader/create_replication.sql +10 -0
- data/scripts/mysql/writer/create_test_db.sql +1 -0
- data/scripts/mysql/writer/create_users.sql +6 -0
- data/scripts/proxysql/proxysql.cnf +117 -0
- data/spec/integration/atomic_switcher_spec.rb +38 -13
- data/spec/integration/chunk_insert_spec.rb +2 -1
- data/spec/integration/chunker_spec.rb +1 -1
- data/spec/integration/database.yml +25 -0
- data/spec/integration/entangler_spec.rb +3 -1
- data/spec/integration/integration_helper.rb +24 -9
- data/spec/integration/lhm_spec.rb +75 -0
- data/spec/integration/proxysql_spec.rb +34 -0
- data/spec/integration/sql_retry/db_connection_helper.rb +52 -0
- data/spec/integration/sql_retry/lock_wait_spec.rb +8 -6
- data/spec/integration/sql_retry/lock_wait_timeout_test_helper.rb +17 -4
- data/spec/integration/sql_retry/proxysql_helper.rb +22 -0
- data/spec/integration/sql_retry/retry_with_proxysql_spec.rb +108 -0
- data/spec/integration/toxiproxy_helper.rb +40 -0
- data/spec/test_helper.rb +21 -0
- data/spec/unit/chunk_insert_spec.rb +7 -2
- data/spec/unit/chunker_spec.rb +46 -42
- data/spec/unit/connection_spec.rb +86 -0
- data/spec/unit/entangler_spec.rb +51 -10
- data/spec/unit/lhm_spec.rb +17 -0
- data/spec/unit/throttler/slave_lag_spec.rb +13 -8
- metadata +85 -14
- data/bin/.gitkeep +0 -0
- data/dbdeployer/config.json +0 -32
- data/dbdeployer/install.sh +0 -64
- data/gemfiles/ar-2.3_mysql.gemfile +0 -6
- data/gemfiles/ar-3.2_mysql.gemfile +0 -5
- data/gemfiles/ar-3.2_mysql2.gemfile +0 -5
- data/gemfiles/ar-4.0_mysql2.gemfile +0 -5
- data/gemfiles/ar-4.1_mysql2.gemfile +0 -5
- data/gemfiles/ar-4.2_mysql2.gemfile +0 -5
- data/gemfiles/ar-5.0_mysql2.gemfile +0 -5
@@ -0,0 +1,22 @@
|
|
1
|
+
class ProxySQLHelper
|
2
|
+
class << self
|
3
|
+
# Flips the destination hostgroup for /maintenance:lhm/ from 0 (i.e. writer) to 1 (i.e. reader)
|
4
|
+
def with_lhm_hostgroup_flip
|
5
|
+
conn = Mysql2::Client.new(
|
6
|
+
host: '127.0.0.1',
|
7
|
+
username: "remote-admin",
|
8
|
+
password: "password",
|
9
|
+
port: "6032",
|
10
|
+
)
|
11
|
+
|
12
|
+
begin
|
13
|
+
conn.query("UPDATE mysql_query_rules SET destination_hostgroup=1 WHERE match_pattern=\"maintenance:lhm\"")
|
14
|
+
conn.query("LOAD MYSQL QUERY RULES TO RUNTIME;")
|
15
|
+
yield
|
16
|
+
ensure
|
17
|
+
conn.query("UPDATE mysql_query_rules SET destination_hostgroup=0 WHERE match_pattern=\"maintenance:lhm\"")
|
18
|
+
conn.query("LOAD MYSQL QUERY RULES TO RUNTIME;")
|
19
|
+
end
|
20
|
+
end
|
21
|
+
end
|
22
|
+
end
|
@@ -0,0 +1,108 @@
|
|
1
|
+
require 'minitest/autorun'
|
2
|
+
require 'mysql2'
|
3
|
+
require 'lhm'
|
4
|
+
require 'toxiproxy'
|
5
|
+
|
6
|
+
require 'integration/sql_retry/lock_wait_timeout_test_helper'
|
7
|
+
require 'integration/sql_retry/db_connection_helper'
|
8
|
+
require 'integration/sql_retry/proxysql_helper'
|
9
|
+
require 'integration/toxiproxy_helper'
|
10
|
+
|
11
|
+
describe Lhm::SqlRetry, "ProxiSQL tests for LHM retry" do
|
12
|
+
include ToxiproxyHelper
|
13
|
+
|
14
|
+
before(:each) do
|
15
|
+
@old_logger = Lhm.logger
|
16
|
+
@logger = StringIO.new
|
17
|
+
Lhm.logger = Logger.new(@logger)
|
18
|
+
|
19
|
+
@connection = DBConnectionHelper::new_mysql_connection(:proxysql, true, true)
|
20
|
+
|
21
|
+
@lhm_retry = Lhm::SqlRetry.new(@connection, retry_options: {},
|
22
|
+
reconnect_with_consistent_host: true)
|
23
|
+
end
|
24
|
+
|
25
|
+
after(:each) do
|
26
|
+
# Restore default logger
|
27
|
+
Lhm.logger = @old_logger
|
28
|
+
end
|
29
|
+
|
30
|
+
it "Will abort if service is down" do
|
31
|
+
|
32
|
+
e = assert_raises Lhm::Error do
|
33
|
+
#Service down
|
34
|
+
Toxiproxy[:mysql_proxysql].down do
|
35
|
+
@lhm_retry.with_retries do |retriable_connection|
|
36
|
+
retriable_connection.execute("INSERT INTO #{DBConnectionHelper.test_table_name} (id) VALUES (2000)")
|
37
|
+
end
|
38
|
+
end
|
39
|
+
end
|
40
|
+
assert_equal Lhm::Error, e.class
|
41
|
+
assert_match(/LHM tried the reconnection procedure but failed. Latest error:/, e.message)
|
42
|
+
end
|
43
|
+
|
44
|
+
it "Will retry until connection is achieved" do
|
45
|
+
|
46
|
+
#Creating a network blip
|
47
|
+
ToxiproxyHelper.with_kill_and_restart(:mysql_proxysql, 2.seconds) do
|
48
|
+
@lhm_retry.with_retries do |retriable_connection|
|
49
|
+
retriable_connection.execute("INSERT INTO #{DBConnectionHelper.test_table_name} (id) VALUES (2000)")
|
50
|
+
end
|
51
|
+
end
|
52
|
+
|
53
|
+
assert_equal @connection.execute("Select * from #{DBConnectionHelper.test_table_name} WHERE id=2000").to_a.first.first, 2000
|
54
|
+
|
55
|
+
logs = @logger.string.split("\n")
|
56
|
+
|
57
|
+
assert logs.first.include?("Lost connection to MySQL, will retry to connect to same host")
|
58
|
+
assert logs.last.include?("LHM successfully reconnected to initial host")
|
59
|
+
end
|
60
|
+
|
61
|
+
it "Will abort if new writer is not same host" do
|
62
|
+
# The hostname will be constant before the blip
|
63
|
+
Lhm::SqlRetry.any_instance.stubs(:hostname).returns("mysql-1").then.returns("mysql-2")
|
64
|
+
|
65
|
+
# Need new instance for stub to take into effect
|
66
|
+
lhm_retry = Lhm::SqlRetry.new(@connection, retry_options: {},
|
67
|
+
reconnect_with_consistent_host: true)
|
68
|
+
|
69
|
+
e = assert_raises Lhm::Error do
|
70
|
+
#Creating a network blip
|
71
|
+
ToxiproxyHelper.with_kill_and_restart(:mysql_proxysql, 2.seconds) do
|
72
|
+
lhm_retry.with_retries do |retriable_connection|
|
73
|
+
retriable_connection.execute("INSERT INTO #{DBConnectionHelper.test_table_name} (id) VALUES (2000)")
|
74
|
+
end
|
75
|
+
end
|
76
|
+
end
|
77
|
+
|
78
|
+
assert_equal e.class, Lhm::Error
|
79
|
+
assert_match(/LHM tried the reconnection procedure but failed. Latest error: Reconnected to wrong host/, e.message)
|
80
|
+
|
81
|
+
logs = @logger.string.split("\n")
|
82
|
+
|
83
|
+
assert logs.first.include?("Lost connection to MySQL, will retry to connect to same host")
|
84
|
+
assert logs.last.include?("Lost connection to MySQL server at 'reading initial communication packet")
|
85
|
+
end
|
86
|
+
|
87
|
+
it "Will abort if failover happens (mimicked with proxySQL)" do
|
88
|
+
e = assert_raises Lhm::Error do
|
89
|
+
#Creates a failover by switching the target hostgroup for the #hostname
|
90
|
+
ProxySQLHelper.with_lhm_hostgroup_flip do
|
91
|
+
#Creating a network blip
|
92
|
+
ToxiproxyHelper.with_kill_and_restart(:mysql_proxysql, 2.seconds) do
|
93
|
+
@lhm_retry.with_retries do |retriable_connection|
|
94
|
+
retriable_connection.execute("INSERT INTO #{DBConnectionHelper.test_table_name} (id) VALUES (2000)")
|
95
|
+
end
|
96
|
+
end
|
97
|
+
end
|
98
|
+
end
|
99
|
+
|
100
|
+
assert_equal e.class, Lhm::Error
|
101
|
+
assert_match(/LHM tried the reconnection procedure but failed. Latest error: Reconnected to wrong host/, e.message)
|
102
|
+
|
103
|
+
logs = @logger.string.split("\n")
|
104
|
+
|
105
|
+
assert logs.first.include?("Lost connection to MySQL, will retry to connect to same host")
|
106
|
+
assert logs.last.include?("Lost connection to MySQL server at 'reading initial communication packet")
|
107
|
+
end
|
108
|
+
end
|
@@ -0,0 +1,40 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
require 'toxiproxy'
|
3
|
+
|
4
|
+
module ToxiproxyHelper
|
5
|
+
class << self
|
6
|
+
|
7
|
+
def included(base)
|
8
|
+
Toxiproxy.reset
|
9
|
+
|
10
|
+
# listen on localhost, but toxiproxy is in a container itself, thus the upstream uses the Docker-Compose DNS
|
11
|
+
Toxiproxy.populate(
|
12
|
+
[
|
13
|
+
{
|
14
|
+
name: 'mysql_master',
|
15
|
+
listen: '0.0.0.0:22220',
|
16
|
+
upstream: 'mysql-1:3306'
|
17
|
+
},
|
18
|
+
{
|
19
|
+
name: 'mysql_proxysql',
|
20
|
+
listen: '0.0.0.0:22222',
|
21
|
+
upstream: 'proxysql:3306'
|
22
|
+
}
|
23
|
+
])
|
24
|
+
end
|
25
|
+
|
26
|
+
def with_kill_and_restart(target, restart_after)
|
27
|
+
thread = Thread.new do
|
28
|
+
sleep(restart_after) unless restart_after.nil?
|
29
|
+
Toxiproxy[target].enable
|
30
|
+
end
|
31
|
+
|
32
|
+
Toxiproxy[target].disable
|
33
|
+
|
34
|
+
yield
|
35
|
+
|
36
|
+
ensure
|
37
|
+
thread.join
|
38
|
+
end
|
39
|
+
end
|
40
|
+
end
|
data/spec/test_helper.rb
CHANGED
@@ -10,6 +10,8 @@ require 'minitest/autorun'
|
|
10
10
|
require 'minitest/spec'
|
11
11
|
require 'minitest/mock'
|
12
12
|
require 'mocha/minitest'
|
13
|
+
require 'after_do'
|
14
|
+
require 'byebug'
|
13
15
|
require 'pathname'
|
14
16
|
require 'lhm'
|
15
17
|
|
@@ -17,6 +19,8 @@ $project = Pathname.new(File.dirname(__FILE__) + '/..').cleanpath
|
|
17
19
|
$spec = $project.join('spec')
|
18
20
|
$fixtures = $spec.join('fixtures')
|
19
21
|
|
22
|
+
$db_name = 'test'
|
23
|
+
|
20
24
|
require 'active_record'
|
21
25
|
require 'mysql2'
|
22
26
|
|
@@ -43,3 +47,20 @@ end
|
|
43
47
|
def throttler
|
44
48
|
Lhm::Throttler::Time.new(:stride => 100)
|
45
49
|
end
|
50
|
+
|
51
|
+
def init_test_db
|
52
|
+
db_config = YAML.load_file(File.expand_path(File.dirname(__FILE__)) + '/integration/database.yml')
|
53
|
+
conn = Mysql2::Client.new(
|
54
|
+
:host => '127.0.0.1',
|
55
|
+
:username => db_config['master']['user'],
|
56
|
+
:password => db_config['master']['password'],
|
57
|
+
:port => db_config['master']['port']
|
58
|
+
)
|
59
|
+
|
60
|
+
conn.query("DROP DATABASE IF EXISTS #{$db_name}")
|
61
|
+
conn.query("CREATE DATABASE #{$db_name}")
|
62
|
+
end
|
63
|
+
|
64
|
+
init_test_db
|
65
|
+
|
66
|
+
|
@@ -4,17 +4,22 @@
|
|
4
4
|
require File.expand_path(File.dirname(__FILE__)) + '/unit_helper'
|
5
5
|
|
6
6
|
require 'lhm/chunk_insert'
|
7
|
+
require 'lhm/connection'
|
7
8
|
|
8
9
|
describe Lhm::ChunkInsert do
|
9
10
|
before(:each) do
|
10
|
-
|
11
|
+
ar_connection = mock()
|
12
|
+
ar_connection.stubs(:execute).returns([["dummy"]])
|
13
|
+
@connection = Lhm::Connection.new(connection: ar_connection, options: {reconnect_with_consistent_host: false})
|
11
14
|
@origin = Lhm::Table.new('foo')
|
12
15
|
@destination = Lhm::Table.new('bar')
|
13
16
|
end
|
14
17
|
|
15
18
|
describe "#sql" do
|
16
19
|
describe "when migration has no conditions" do
|
17
|
-
before
|
20
|
+
before do
|
21
|
+
@migration = Lhm::Migration.new(@origin, @destination)
|
22
|
+
end
|
18
23
|
|
19
24
|
it "uses a simple where clause" do
|
20
25
|
assert_equal(
|
data/spec/unit/chunker_spec.rb
CHANGED
@@ -7,15 +7,19 @@ require 'lhm/table'
|
|
7
7
|
require 'lhm/migration'
|
8
8
|
require 'lhm/chunker'
|
9
9
|
require 'lhm/throttler'
|
10
|
+
require 'lhm/connection'
|
10
11
|
|
11
12
|
describe Lhm::Chunker do
|
12
13
|
include UnitHelper
|
13
14
|
|
15
|
+
EXPECTED_RETRY_FLAGS = {:should_retry => true, :retry_options => {}}
|
16
|
+
|
14
17
|
before(:each) do
|
15
18
|
@origin = Lhm::Table.new('foo')
|
16
19
|
@destination = Lhm::Table.new('bar')
|
17
20
|
@migration = Lhm::Migration.new(@origin, @destination)
|
18
21
|
@connection = mock()
|
22
|
+
@connection.stubs(:execute).returns([["dummy"]])
|
19
23
|
# This is a poor man's stub
|
20
24
|
@throttler = Object.new
|
21
25
|
def @throttler.run
|
@@ -37,11 +41,11 @@ describe Lhm::Chunker do
|
|
37
41
|
5
|
38
42
|
end
|
39
43
|
|
40
|
-
@connection.expects(:select_value).with(regexp_matches(/where id >= 1 order by id limit 1 offset 4/)).returns(7)
|
41
|
-
@connection.expects(:select_value).with(regexp_matches(/where id >= 8 order by id limit 1 offset 4/)).returns(21)
|
42
|
-
@connection.expects(:update).with(regexp_matches(/between 1 and 7/)).returns(2)
|
43
|
-
@connection.expects(:update).with(regexp_matches(/between 8 and 10/)).returns(2)
|
44
|
-
@connection.expects(:
|
44
|
+
@connection.expects(:select_value).with(regexp_matches(/where id >= 1 order by id limit 1 offset 4/),EXPECTED_RETRY_FLAGS).returns(7)
|
45
|
+
@connection.expects(:select_value).with(regexp_matches(/where id >= 8 order by id limit 1 offset 4/),EXPECTED_RETRY_FLAGS).returns(21)
|
46
|
+
@connection.expects(:update).with(regexp_matches(/between 1 and 7/),EXPECTED_RETRY_FLAGS).returns(2)
|
47
|
+
@connection.expects(:update).with(regexp_matches(/between 8 and 10/),EXPECTED_RETRY_FLAGS).returns(2)
|
48
|
+
@connection.expects(:execute).twice.with(regexp_matches(/show warnings/),EXPECTED_RETRY_FLAGS).returns([])
|
45
49
|
|
46
50
|
@chunker.run
|
47
51
|
end
|
@@ -52,17 +56,17 @@ describe Lhm::Chunker do
|
|
52
56
|
2
|
53
57
|
end
|
54
58
|
|
55
|
-
@connection.expects(:select_value).with(regexp_matches(/where id >= 1 order by id limit 1 offset 1/)).returns(2)
|
56
|
-
@connection.expects(:select_value).with(regexp_matches(/where id >= 3 order by id limit 1 offset 1/)).returns(4)
|
57
|
-
@connection.expects(:select_value).with(regexp_matches(/where id >= 5 order by id limit 1 offset 1/)).returns(6)
|
58
|
-
@connection.expects(:select_value).with(regexp_matches(/where id >= 7 order by id limit 1 offset 1/)).returns(8)
|
59
|
-
@connection.expects(:select_value).with(regexp_matches(/where id >= 9 order by id limit 1 offset 1/)).returns(10)
|
59
|
+
@connection.expects(:select_value).with(regexp_matches(/where id >= 1 order by id limit 1 offset 1/),EXPECTED_RETRY_FLAGS).returns(2)
|
60
|
+
@connection.expects(:select_value).with(regexp_matches(/where id >= 3 order by id limit 1 offset 1/),EXPECTED_RETRY_FLAGS).returns(4)
|
61
|
+
@connection.expects(:select_value).with(regexp_matches(/where id >= 5 order by id limit 1 offset 1/),EXPECTED_RETRY_FLAGS).returns(6)
|
62
|
+
@connection.expects(:select_value).with(regexp_matches(/where id >= 7 order by id limit 1 offset 1/),EXPECTED_RETRY_FLAGS).returns(8)
|
63
|
+
@connection.expects(:select_value).with(regexp_matches(/where id >= 9 order by id limit 1 offset 1/),EXPECTED_RETRY_FLAGS).returns(10)
|
60
64
|
|
61
|
-
@connection.expects(:update).with(regexp_matches(/between 1 and 2/)).returns(2)
|
62
|
-
@connection.expects(:update).with(regexp_matches(/between 3 and 4/)).returns(2)
|
63
|
-
@connection.expects(:update).with(regexp_matches(/between 5 and 6/)).returns(2)
|
64
|
-
@connection.expects(:update).with(regexp_matches(/between 7 and 8/)).returns(2)
|
65
|
-
@connection.expects(:update).with(regexp_matches(/between 9 and 10/)).returns(2)
|
65
|
+
@connection.expects(:update).with(regexp_matches(/between 1 and 2/),EXPECTED_RETRY_FLAGS).returns(2)
|
66
|
+
@connection.expects(:update).with(regexp_matches(/between 3 and 4/),EXPECTED_RETRY_FLAGS).returns(2)
|
67
|
+
@connection.expects(:update).with(regexp_matches(/between 5 and 6/),EXPECTED_RETRY_FLAGS).returns(2)
|
68
|
+
@connection.expects(:update).with(regexp_matches(/between 7 and 8/),EXPECTED_RETRY_FLAGS).returns(2)
|
69
|
+
@connection.expects(:update).with(regexp_matches(/between 9 and 10/),EXPECTED_RETRY_FLAGS).returns(2)
|
66
70
|
|
67
71
|
@chunker.run
|
68
72
|
end
|
@@ -79,17 +83,17 @@ describe Lhm::Chunker do
|
|
79
83
|
end
|
80
84
|
end
|
81
85
|
|
82
|
-
@connection.expects(:select_value).with(regexp_matches(/where id >= 1 order by id limit 1 offset 1/)).returns(2)
|
83
|
-
@connection.expects(:select_value).with(regexp_matches(/where id >= 3 order by id limit 1 offset 2/)).returns(5)
|
84
|
-
@connection.expects(:select_value).with(regexp_matches(/where id >= 6 order by id limit 1 offset 2/)).returns(8)
|
85
|
-
@connection.expects(:select_value).with(regexp_matches(/where id >= 9 order by id limit 1 offset 2/)).returns(nil)
|
86
|
+
@connection.expects(:select_value).with(regexp_matches(/where id >= 1 order by id limit 1 offset 1/),EXPECTED_RETRY_FLAGS).returns(2)
|
87
|
+
@connection.expects(:select_value).with(regexp_matches(/where id >= 3 order by id limit 1 offset 2/),EXPECTED_RETRY_FLAGS).returns(5)
|
88
|
+
@connection.expects(:select_value).with(regexp_matches(/where id >= 6 order by id limit 1 offset 2/),EXPECTED_RETRY_FLAGS).returns(8)
|
89
|
+
@connection.expects(:select_value).with(regexp_matches(/where id >= 9 order by id limit 1 offset 2/),EXPECTED_RETRY_FLAGS).returns(nil)
|
86
90
|
|
87
|
-
@connection.expects(:update).with(regexp_matches(/between 1 and 2/)).returns(2)
|
88
|
-
@connection.expects(:update).with(regexp_matches(/between 3 and 5/)).returns(2)
|
89
|
-
@connection.expects(:update).with(regexp_matches(/between 6 and 8/)).returns(2)
|
90
|
-
@connection.expects(:update).with(regexp_matches(/between 9 and 10/)).returns(2)
|
91
|
+
@connection.expects(:update).with(regexp_matches(/between 1 and 2/),EXPECTED_RETRY_FLAGS).returns(2)
|
92
|
+
@connection.expects(:update).with(regexp_matches(/between 3 and 5/),EXPECTED_RETRY_FLAGS).returns(2)
|
93
|
+
@connection.expects(:update).with(regexp_matches(/between 6 and 8/),EXPECTED_RETRY_FLAGS).returns(2)
|
94
|
+
@connection.expects(:update).with(regexp_matches(/between 9 and 10/),EXPECTED_RETRY_FLAGS).returns(2)
|
91
95
|
|
92
|
-
@connection.expects(:
|
96
|
+
@connection.expects(:execute).twice.with(regexp_matches(/show warnings/),EXPECTED_RETRY_FLAGS).returns([])
|
93
97
|
|
94
98
|
@chunker.run
|
95
99
|
end
|
@@ -99,8 +103,8 @@ describe Lhm::Chunker do
|
|
99
103
|
:start => 1,
|
100
104
|
:limit => 1)
|
101
105
|
|
102
|
-
@connection.expects(:select_value).with(regexp_matches(/where id >= 1 order by id limit 1 offset 0/)).returns(nil)
|
103
|
-
@connection.expects(:update).with(regexp_matches(/between 1 and 1/)).returns(1)
|
106
|
+
@connection.expects(:select_value).with(regexp_matches(/where id >= 1 order by id limit 1 offset 0/),EXPECTED_RETRY_FLAGS).returns(nil)
|
107
|
+
@connection.expects(:update).with(regexp_matches(/between 1 and 1/),EXPECTED_RETRY_FLAGS).returns(1)
|
104
108
|
|
105
109
|
@chunker.run
|
106
110
|
end
|
@@ -113,17 +117,17 @@ describe Lhm::Chunker do
|
|
113
117
|
2
|
114
118
|
end
|
115
119
|
|
116
|
-
@connection.expects(:select_value).with(regexp_matches(/where id >= 2 order by id limit 1 offset 1/)).returns(3)
|
117
|
-
@connection.expects(:select_value).with(regexp_matches(/where id >= 4 order by id limit 1 offset 1/)).returns(5)
|
118
|
-
@connection.expects(:select_value).with(regexp_matches(/where id >= 6 order by id limit 1 offset 1/)).returns(7)
|
119
|
-
@connection.expects(:select_value).with(regexp_matches(/where id >= 8 order by id limit 1 offset 1/)).returns(9)
|
120
|
-
@connection.expects(:select_value).with(regexp_matches(/where id >= 10 order by id limit 1 offset 1/)).returns(nil)
|
120
|
+
@connection.expects(:select_value).with(regexp_matches(/where id >= 2 order by id limit 1 offset 1/),EXPECTED_RETRY_FLAGS).returns(3)
|
121
|
+
@connection.expects(:select_value).with(regexp_matches(/where id >= 4 order by id limit 1 offset 1/),EXPECTED_RETRY_FLAGS).returns(5)
|
122
|
+
@connection.expects(:select_value).with(regexp_matches(/where id >= 6 order by id limit 1 offset 1/),EXPECTED_RETRY_FLAGS).returns(7)
|
123
|
+
@connection.expects(:select_value).with(regexp_matches(/where id >= 8 order by id limit 1 offset 1/),EXPECTED_RETRY_FLAGS).returns(9)
|
124
|
+
@connection.expects(:select_value).with(regexp_matches(/where id >= 10 order by id limit 1 offset 1/),EXPECTED_RETRY_FLAGS).returns(nil)
|
121
125
|
|
122
|
-
@connection.expects(:update).with(regexp_matches(/between 2 and 3/)).returns(2)
|
123
|
-
@connection.expects(:update).with(regexp_matches(/between 4 and 5/)).returns(2)
|
124
|
-
@connection.expects(:update).with(regexp_matches(/between 6 and 7/)).returns(2)
|
125
|
-
@connection.expects(:update).with(regexp_matches(/between 8 and 9/)).returns(2)
|
126
|
-
@connection.expects(:update).with(regexp_matches(/between 10 and 10/)).returns(1)
|
126
|
+
@connection.expects(:update).with(regexp_matches(/between 2 and 3/),EXPECTED_RETRY_FLAGS).returns(2)
|
127
|
+
@connection.expects(:update).with(regexp_matches(/between 4 and 5/),EXPECTED_RETRY_FLAGS).returns(2)
|
128
|
+
@connection.expects(:update).with(regexp_matches(/between 6 and 7/),EXPECTED_RETRY_FLAGS).returns(2)
|
129
|
+
@connection.expects(:update).with(regexp_matches(/between 8 and 9/),EXPECTED_RETRY_FLAGS).returns(2)
|
130
|
+
@connection.expects(:update).with(regexp_matches(/between 10 and 10/),EXPECTED_RETRY_FLAGS).returns(1)
|
127
131
|
|
128
132
|
@chunker.run
|
129
133
|
end
|
@@ -137,9 +141,9 @@ describe Lhm::Chunker do
|
|
137
141
|
2
|
138
142
|
end
|
139
143
|
|
140
|
-
@connection.expects(:select_value).with(regexp_matches(/where id >= 1 order by id limit 1 offset 1/)).returns(2)
|
141
|
-
@connection.expects(:update).with(regexp_matches(/where \(foo.created_at > '2013-07-10' or foo.baz = 'quux'\) and `foo`/)).returns(1)
|
142
|
-
@connection.expects(:
|
144
|
+
@connection.expects(:select_value).with(regexp_matches(/where id >= 1 order by id limit 1 offset 1/),EXPECTED_RETRY_FLAGS).returns(2)
|
145
|
+
@connection.expects(:update).with(regexp_matches(/where \(foo.created_at > '2013-07-10' or foo.baz = 'quux'\) and `foo`/),EXPECTED_RETRY_FLAGS).returns(1)
|
146
|
+
@connection.expects(:execute).with(regexp_matches(/show warnings/),EXPECTED_RETRY_FLAGS).returns([])
|
143
147
|
|
144
148
|
def @migration.conditions
|
145
149
|
"where foo.created_at > '2013-07-10' or foo.baz = 'quux'"
|
@@ -157,9 +161,9 @@ describe Lhm::Chunker do
|
|
157
161
|
2
|
158
162
|
end
|
159
163
|
|
160
|
-
@connection.expects(:select_value).with(regexp_matches(/where id >= 1 order by id limit 1 offset 1/)).returns(2)
|
161
|
-
@connection.expects(:update).with(regexp_matches(/inner join bar on foo.id = bar.foo_id and/)).returns(1)
|
162
|
-
@connection.expects(:
|
164
|
+
@connection.expects(:select_value).with(regexp_matches(/where id >= 1 order by id limit 1 offset 1/),EXPECTED_RETRY_FLAGS).returns(2)
|
165
|
+
@connection.expects(:update).with(regexp_matches(/inner join bar on foo.id = bar.foo_id and/),EXPECTED_RETRY_FLAGS).returns(1)
|
166
|
+
@connection.expects(:execute).with(regexp_matches(/show warnings/),EXPECTED_RETRY_FLAGS).returns([])
|
163
167
|
|
164
168
|
def @migration.conditions
|
165
169
|
'inner join bar on foo.id = bar.foo_id'
|
@@ -0,0 +1,86 @@
|
|
1
|
+
require 'lhm/connection'
|
2
|
+
require 'lhm/proxysql_helper'
|
3
|
+
|
4
|
+
describe Lhm::Connection do
|
5
|
+
|
6
|
+
LOCK_WAIT = ActiveRecord::StatementInvalid.new('Lock wait timeout exceeded; try restarting transaction.')
|
7
|
+
|
8
|
+
before(:each) do
|
9
|
+
@logs = StringIO.new
|
10
|
+
Lhm.logger = Logger.new(@logs)
|
11
|
+
end
|
12
|
+
|
13
|
+
it "Should find use calling file as prefix" do
|
14
|
+
ar_connection = mock()
|
15
|
+
ar_connection.stubs(:execute).raises(LOCK_WAIT).then.returns(true)
|
16
|
+
ar_connection.stubs(:active?).returns(true)
|
17
|
+
|
18
|
+
connection = Lhm::Connection.new(connection: ar_connection)
|
19
|
+
|
20
|
+
connection.execute("SHOW TABLES", should_retry: true, retry_options: { base_interval: 0 })
|
21
|
+
|
22
|
+
log_messages = @logs.string.split("\n")
|
23
|
+
assert_equal(1, log_messages.length)
|
24
|
+
assert log_messages.first.include?("[ConnectionSpec]")
|
25
|
+
end
|
26
|
+
|
27
|
+
it "#execute should be retried" do
|
28
|
+
ar_connection = mock()
|
29
|
+
ar_connection.stubs(:execute).raises(LOCK_WAIT)
|
30
|
+
.then.raises(LOCK_WAIT)
|
31
|
+
.then.returns(true)
|
32
|
+
ar_connection.stubs(:active?).returns(true)
|
33
|
+
|
34
|
+
connection = Lhm::Connection.new(connection: ar_connection)
|
35
|
+
|
36
|
+
connection.execute("SHOW TABLES", should_retry: true, retry_options: { base_interval: 0, tries: 3 })
|
37
|
+
|
38
|
+
log_messages = @logs.string.split("\n")
|
39
|
+
assert_equal(2, log_messages.length)
|
40
|
+
end
|
41
|
+
|
42
|
+
it "#update should be retried" do
|
43
|
+
ar_connection = mock()
|
44
|
+
ar_connection.stubs(:update).raises(LOCK_WAIT)
|
45
|
+
.then.raises(LOCK_WAIT)
|
46
|
+
.then.returns(1)
|
47
|
+
ar_connection.stubs(:active?).returns(true)
|
48
|
+
|
49
|
+
connection = Lhm::Connection.new(connection: ar_connection)
|
50
|
+
|
51
|
+
val = connection.update("SHOW TABLES", should_retry: true, retry_options:{ base_interval: 0, tries: 3 })
|
52
|
+
|
53
|
+
log_messages = @logs.string.split("\n")
|
54
|
+
assert_equal val, 1
|
55
|
+
assert_equal(2, log_messages.length)
|
56
|
+
end
|
57
|
+
|
58
|
+
it "#select_value should be retried" do
|
59
|
+
ar_connection = mock()
|
60
|
+
ar_connection.stubs(:select_value).raises(LOCK_WAIT)
|
61
|
+
.then.raises(LOCK_WAIT)
|
62
|
+
.then.returns("dummy")
|
63
|
+
ar_connection.stubs(:active?).returns(true)
|
64
|
+
|
65
|
+
connection = Lhm::Connection.new(connection: ar_connection)
|
66
|
+
|
67
|
+
val = connection.select_value("SHOW TABLES", should_retry: true, retry_options: { base_interval: 0, tries: 3 })
|
68
|
+
|
69
|
+
log_messages = @logs.string.split("\n")
|
70
|
+
assert_equal val, "dummy"
|
71
|
+
assert_equal(2, log_messages.length)
|
72
|
+
end
|
73
|
+
|
74
|
+
it "Queries should be tagged with ProxySQL tag if requested" do
|
75
|
+
ar_connection = mock()
|
76
|
+
ar_connection.expects(:public_send).with(:select_value, "#{Lhm::ProxySQLHelper::ANNOTATION}SHOW TABLES").returns("dummy")
|
77
|
+
ar_connection.stubs(:execute).times(4).returns([["dummy"]])
|
78
|
+
ar_connection.stubs(:active?).returns(true)
|
79
|
+
|
80
|
+
connection = Lhm::Connection.new(connection: ar_connection, options: { reconnect_with_consistent_host: true })
|
81
|
+
|
82
|
+
val = connection.select_value("SHOW TABLES", should_retry: true, retry_options: { base_interval: 0, tries: 3 })
|
83
|
+
|
84
|
+
assert_equal val, "dummy"
|
85
|
+
end
|
86
|
+
end
|
data/spec/unit/entangler_spec.rb
CHANGED
@@ -6,6 +6,7 @@ require File.expand_path(File.dirname(__FILE__)) + '/unit_helper'
|
|
6
6
|
require 'lhm/table'
|
7
7
|
require 'lhm/migration'
|
8
8
|
require 'lhm/entangler'
|
9
|
+
require 'lhm/connection'
|
9
10
|
|
10
11
|
describe Lhm::Entangler do
|
11
12
|
include UnitHelper
|
@@ -60,33 +61,73 @@ describe Lhm::Entangler do
|
|
60
61
|
end
|
61
62
|
|
62
63
|
it 'should retry trigger creation when it hits a lock wait timeout' do
|
63
|
-
connection = mock()
|
64
64
|
tries = 1
|
65
|
+
ar_connection = mock()
|
66
|
+
ar_connection.stubs(:execute)
|
67
|
+
.returns([["dummy"]], [["dummy"]], [["dummy"]])
|
68
|
+
.then
|
69
|
+
.raises(Mysql2::Error, 'Lock wait timeout exceeded; try restarting transaction')
|
70
|
+
ar_connection.stubs(:active?).returns(true)
|
71
|
+
|
72
|
+
connection = Lhm::Connection.new(connection: ar_connection, options: {reconnect_with_consistent_host: true})
|
73
|
+
|
65
74
|
@entangler = Lhm::Entangler.new(@migration, connection, retriable: {base_interval: 0, tries: tries})
|
66
|
-
connection.expects(:execute).times(tries).raises(Mysql2::Error, 'Lock wait timeout exceeded; try restarting transaction')
|
67
75
|
|
68
76
|
assert_raises(Mysql2::Error) { @entangler.before }
|
69
77
|
end
|
70
78
|
|
71
79
|
it 'should not retry trigger creation with other mysql errors' do
|
72
|
-
|
73
|
-
|
74
|
-
|
75
|
-
|
80
|
+
ar_connection = mock()
|
81
|
+
ar_connection.stubs(:execute)
|
82
|
+
.returns([["dummy"]], [["dummy"]], [["dummy"]])
|
83
|
+
.then
|
84
|
+
.raises(Mysql2::Error, 'The MySQL server is running with the --read-only option so it cannot execute this statement.')
|
85
|
+
ar_connection.stubs(:active?).returns(true)
|
86
|
+
connection = Lhm::Connection.new(connection: ar_connection, options: {reconnect_with_consistent_host: true})
|
87
|
+
|
88
|
+
@entangler = Lhm::Entangler.new(@migration, connection, retriable: { base_interval: 0 })
|
76
89
|
assert_raises(Mysql2::Error) { @entangler.before }
|
77
90
|
end
|
78
91
|
|
79
92
|
it 'should succesfully finish after retrying' do
|
80
|
-
|
81
|
-
|
93
|
+
ar_connection = mock()
|
94
|
+
ar_connection.stubs(:execute)
|
95
|
+
.returns([["dummy"]], [["dummy"]], [["dummy"]])
|
96
|
+
.then
|
97
|
+
.raises(Mysql2::Error, 'Lock wait timeout exceeded; try restarting transaction')
|
98
|
+
.then
|
99
|
+
.returns([["dummy"]])
|
100
|
+
ar_connection.stubs(:active?).returns(true)
|
101
|
+
|
102
|
+
connection = Lhm::Connection.new(connection: ar_connection, options: {reconnect_with_consistent_host: true})
|
103
|
+
|
82
104
|
@entangler = Lhm::Entangler.new(@migration, connection, retriable: {base_interval: 0})
|
83
105
|
|
84
106
|
assert @entangler.before
|
85
107
|
end
|
86
108
|
|
87
109
|
it 'should retry as many times as specified by configuration' do
|
88
|
-
|
89
|
-
|
110
|
+
ar_connection = mock()
|
111
|
+
ar_connection.stubs(:execute)
|
112
|
+
.returns([["dummy"]], [["dummy"]], [["dummy"]]) # initial
|
113
|
+
.then
|
114
|
+
.raises(Mysql2::Error, 'Lock wait timeout exceeded; try restarting transaction')
|
115
|
+
.then
|
116
|
+
.returns([["dummy"]]) # reconnect 1
|
117
|
+
.then
|
118
|
+
.raises(Mysql2::Error, 'Lock wait timeout exceeded; try restarting transaction')
|
119
|
+
.then
|
120
|
+
.returns([["dummy"]]) # reconnect 2
|
121
|
+
.then
|
122
|
+
.raises(Mysql2::Error, 'Lock wait timeout exceeded; try restarting transaction')
|
123
|
+
.then
|
124
|
+
.returns([["dummy"]]) # reconnect 3
|
125
|
+
.then
|
126
|
+
.raises(Mysql2::Error, 'Lock wait timeout exceeded; try restarting transaction') # final error
|
127
|
+
ar_connection.stubs(:active?).returns(true)
|
128
|
+
|
129
|
+
connection = Lhm::Connection.new(connection: ar_connection, options: {reconnect_with_consistent_host: true})
|
130
|
+
|
90
131
|
@entangler = Lhm::Entangler.new(@migration, connection, retriable: {tries: 5, base_interval: 0})
|
91
132
|
|
92
133
|
assert_raises(Mysql2::Error) { @entangler.before }
|
data/spec/unit/lhm_spec.rb
CHANGED
@@ -26,4 +26,21 @@ describe Lhm do
|
|
26
26
|
value(Lhm.logger.instance_eval { @logdev }.dev.path).must_equal 'omg.ponies'
|
27
27
|
end
|
28
28
|
end
|
29
|
+
|
30
|
+
describe 'api' do
|
31
|
+
|
32
|
+
before(:each) do
|
33
|
+
@connection = mock()
|
34
|
+
end
|
35
|
+
|
36
|
+
it 'should create a new connection when calling setup' do
|
37
|
+
Lhm.setup(@connection)
|
38
|
+
value(Lhm.connection).must_be_kind_of(Lhm::Connection)
|
39
|
+
end
|
40
|
+
|
41
|
+
it 'should create a new connection when none is created' do
|
42
|
+
ActiveRecord::Base.stubs(:connection).returns(@connection)
|
43
|
+
value(Lhm.connection).must_be_kind_of(Lhm::Connection)
|
44
|
+
end
|
45
|
+
end
|
29
46
|
end
|
@@ -39,7 +39,7 @@ describe Lhm::Throttler::Slave do
|
|
39
39
|
@logs = StringIO.new
|
40
40
|
Lhm.logger = Logger.new(@logs)
|
41
41
|
|
42
|
-
@dummy_mysql_client_config = lambda { {'username' => 'user', 'password' => 'pw', 'database' => 'db'} }
|
42
|
+
@dummy_mysql_client_config = lambda { { 'username' => 'user', 'password' => 'pw', 'database' => 'db' } }
|
43
43
|
end
|
44
44
|
|
45
45
|
describe "#client" do
|
@@ -64,7 +64,7 @@ describe Lhm::Throttler::Slave do
|
|
64
64
|
|
65
65
|
describe 'with proper config' do
|
66
66
|
it "creates a new Mysql2::Client" do
|
67
|
-
expected_config = {username: 'user', password: 'pw', database: 'db', host: 'slave'}
|
67
|
+
expected_config = { username: 'user', password: 'pw', database: 'db', host: 'slave' }
|
68
68
|
Mysql2::Client.stubs(:new).with(expected_config).returns(mock())
|
69
69
|
|
70
70
|
assert Lhm::Throttler::Slave.new('slave', @dummy_mysql_client_config).connection
|
@@ -73,8 +73,12 @@ describe Lhm::Throttler::Slave do
|
|
73
73
|
|
74
74
|
describe 'with active record config' do
|
75
75
|
it 'logs and creates client' do
|
76
|
-
active_record_config = {username: 'user', password: 'pw', database: 'db'}
|
77
|
-
ActiveRecord::
|
76
|
+
active_record_config = { username: 'user', password: 'pw', database: 'db' }
|
77
|
+
if ActiveRecord::VERSION::MAJOR > 6 || ActiveRecord::VERSION::MAJOR == 6 && ActiveRecord::VERSION::MINOR >= 1
|
78
|
+
ActiveRecord::Base.stubs(:connection_pool).returns(stub(db_config: stub(configuration_hash: active_record_config)))
|
79
|
+
else
|
80
|
+
ActiveRecord::Base.stubs(:connection_pool).returns(stub(spec: stub(config: active_record_config)))
|
81
|
+
end
|
78
82
|
|
79
83
|
Mysql2::Client.stubs(:new).returns(mock())
|
80
84
|
|
@@ -92,9 +96,9 @@ describe Lhm::Throttler::Slave do
|
|
92
96
|
class Connection
|
93
97
|
def self.query(query)
|
94
98
|
if query == Lhm::Throttler::Slave::SQL_SELECT_MAX_SLAVE_LAG
|
95
|
-
[{'Seconds_Behind_Master' => 20}]
|
99
|
+
[{ 'Seconds_Behind_Master' => 20 }]
|
96
100
|
elsif query == Lhm::Throttler::Slave::SQL_SELECT_SLAVE_HOSTS
|
97
|
-
[{'host' => '1.1.1.1:80'}]
|
101
|
+
[{ 'host' => '1.1.1.1:80' }]
|
98
102
|
end
|
99
103
|
end
|
100
104
|
end
|
@@ -104,7 +108,7 @@ describe Lhm::Throttler::Slave do
|
|
104
108
|
|
105
109
|
class StoppedConnection
|
106
110
|
def self.query(query)
|
107
|
-
[{'Seconds_Behind_Master' => nil}]
|
111
|
+
[{ 'Seconds_Behind_Master' => nil }]
|
108
112
|
end
|
109
113
|
end
|
110
114
|
|
@@ -286,7 +290,7 @@ describe Lhm::Throttler::SlaveLag do
|
|
286
290
|
describe 'with the :check_only option' do
|
287
291
|
describe 'with a callable argument' do
|
288
292
|
before do
|
289
|
-
check_only = lambda {{'host' => '1.1.1.3'}}
|
293
|
+
check_only = lambda { { 'host' => '1.1.1.3' } }
|
290
294
|
@throttler = Lhm::Throttler::SlaveLag.new :check_only => check_only
|
291
295
|
end
|
292
296
|
|
@@ -300,6 +304,7 @@ describe Lhm::Throttler::SlaveLag do
|
|
300
304
|
describe 'with a non-callable argument' do
|
301
305
|
before do
|
302
306
|
@throttler = Lhm::Throttler::SlaveLag.new :check_only => 'I cannot be called'
|
307
|
+
|
303
308
|
def @throttler.master_slave_hosts
|
304
309
|
['1.1.1.1', '1.1.1.4']
|
305
310
|
end
|