lhm-shopify 3.5.0 → 3.5.4
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.github/workflows/test.yml +17 -4
- data/.gitignore +0 -2
- data/Appraisals +24 -0
- data/CHANGELOG.md +23 -0
- data/Gemfile.lock +66 -0
- data/README.md +53 -0
- data/Rakefile +6 -5
- data/dev.yml +18 -3
- data/docker-compose.yml +15 -3
- data/gemfiles/activerecord_5.2.gemfile +9 -0
- data/gemfiles/activerecord_5.2.gemfile.lock +65 -0
- data/gemfiles/activerecord_6.0.gemfile +7 -0
- data/gemfiles/activerecord_6.0.gemfile.lock +67 -0
- data/gemfiles/activerecord_6.1.gemfile +7 -0
- data/gemfiles/activerecord_6.1.gemfile.lock +66 -0
- data/gemfiles/activerecord_7.0.0.alpha2.gemfile +7 -0
- data/gemfiles/activerecord_7.0.0.alpha2.gemfile.lock +64 -0
- data/lhm.gemspec +7 -3
- data/lib/lhm/atomic_switcher.rb +4 -3
- data/lib/lhm/chunk_insert.rb +7 -3
- data/lib/lhm/chunker.rb +6 -6
- data/lib/lhm/cleanup/current.rb +4 -1
- data/lib/lhm/connection.rb +66 -19
- data/lib/lhm/entangler.rb +5 -4
- data/lib/lhm/invoker.rb +5 -3
- data/lib/lhm/locked_switcher.rb +2 -0
- data/lib/lhm/proxysql_helper.rb +10 -0
- data/lib/lhm/sql_retry.rb +135 -11
- data/lib/lhm/throttler/slave_lag.rb +19 -2
- data/lib/lhm/version.rb +1 -1
- data/lib/lhm.rb +32 -12
- data/scripts/mysql/writer/create_users.sql +3 -0
- data/spec/integration/atomic_switcher_spec.rb +38 -10
- data/spec/integration/chunk_insert_spec.rb +2 -1
- data/spec/integration/chunker_spec.rb +8 -6
- data/spec/integration/database.yml +10 -0
- data/spec/integration/entangler_spec.rb +3 -1
- data/spec/integration/integration_helper.rb +20 -4
- data/spec/integration/lhm_spec.rb +75 -0
- data/spec/integration/proxysql_spec.rb +34 -0
- data/spec/integration/sql_retry/db_connection_helper.rb +52 -0
- data/spec/integration/sql_retry/lock_wait_spec.rb +8 -6
- data/spec/integration/sql_retry/lock_wait_timeout_test_helper.rb +19 -9
- data/spec/integration/sql_retry/proxysql_helper.rb +22 -0
- data/spec/integration/sql_retry/retry_with_proxysql_spec.rb +108 -0
- data/spec/integration/toxiproxy_helper.rb +40 -0
- data/spec/test_helper.rb +21 -0
- data/spec/unit/chunk_insert_spec.rb +7 -2
- data/spec/unit/chunker_spec.rb +46 -42
- data/spec/unit/connection_spec.rb +51 -8
- data/spec/unit/entangler_spec.rb +71 -19
- data/spec/unit/lhm_spec.rb +17 -0
- data/spec/unit/throttler/slave_lag_spec.rb +14 -9
- metadata +76 -11
- data/gemfiles/ar-2.3_mysql.gemfile +0 -6
- data/gemfiles/ar-3.2_mysql.gemfile +0 -5
- data/gemfiles/ar-3.2_mysql2.gemfile +0 -5
- data/gemfiles/ar-4.0_mysql2.gemfile +0 -5
- data/gemfiles/ar-4.1_mysql2.gemfile +0 -5
- data/gemfiles/ar-4.2_mysql2.gemfile +0 -5
- data/gemfiles/ar-5.0_mysql2.gemfile +0 -5
data/spec/test_helper.rb
CHANGED
@@ -10,6 +10,8 @@ require 'minitest/autorun'
|
|
10
10
|
require 'minitest/spec'
|
11
11
|
require 'minitest/mock'
|
12
12
|
require 'mocha/minitest'
|
13
|
+
require 'after_do'
|
14
|
+
require 'byebug'
|
13
15
|
require 'pathname'
|
14
16
|
require 'lhm'
|
15
17
|
|
@@ -17,6 +19,8 @@ $project = Pathname.new(File.dirname(__FILE__) + '/..').cleanpath
|
|
17
19
|
$spec = $project.join('spec')
|
18
20
|
$fixtures = $spec.join('fixtures')
|
19
21
|
|
22
|
+
$db_name = 'test'
|
23
|
+
|
20
24
|
require 'active_record'
|
21
25
|
require 'mysql2'
|
22
26
|
|
@@ -43,3 +47,20 @@ end
|
|
43
47
|
def throttler
|
44
48
|
Lhm::Throttler::Time.new(:stride => 100)
|
45
49
|
end
|
50
|
+
|
51
|
+
def init_test_db
|
52
|
+
db_config = YAML.load_file(File.expand_path(File.dirname(__FILE__)) + '/integration/database.yml')
|
53
|
+
conn = Mysql2::Client.new(
|
54
|
+
:host => '127.0.0.1',
|
55
|
+
:username => db_config['master']['user'],
|
56
|
+
:password => db_config['master']['password'],
|
57
|
+
:port => db_config['master']['port']
|
58
|
+
)
|
59
|
+
|
60
|
+
conn.query("DROP DATABASE IF EXISTS #{$db_name}")
|
61
|
+
conn.query("CREATE DATABASE #{$db_name}")
|
62
|
+
end
|
63
|
+
|
64
|
+
init_test_db
|
65
|
+
|
66
|
+
|
@@ -4,17 +4,22 @@
|
|
4
4
|
require File.expand_path(File.dirname(__FILE__)) + '/unit_helper'
|
5
5
|
|
6
6
|
require 'lhm/chunk_insert'
|
7
|
+
require 'lhm/connection'
|
7
8
|
|
8
9
|
describe Lhm::ChunkInsert do
|
9
10
|
before(:each) do
|
10
|
-
|
11
|
+
ar_connection = mock()
|
12
|
+
ar_connection.stubs(:execute).returns([["dummy"]])
|
13
|
+
@connection = Lhm::Connection.new(connection: ar_connection, options: {reconnect_with_consistent_host: false})
|
11
14
|
@origin = Lhm::Table.new('foo')
|
12
15
|
@destination = Lhm::Table.new('bar')
|
13
16
|
end
|
14
17
|
|
15
18
|
describe "#sql" do
|
16
19
|
describe "when migration has no conditions" do
|
17
|
-
before
|
20
|
+
before do
|
21
|
+
@migration = Lhm::Migration.new(@origin, @destination)
|
22
|
+
end
|
18
23
|
|
19
24
|
it "uses a simple where clause" do
|
20
25
|
assert_equal(
|
data/spec/unit/chunker_spec.rb
CHANGED
@@ -12,11 +12,15 @@ require 'lhm/connection'
|
|
12
12
|
describe Lhm::Chunker do
|
13
13
|
include UnitHelper
|
14
14
|
|
15
|
+
EXPECTED_RETRY_FLAGS_CHUNKER = {:should_retry => true, :log_prefix => "Chunker"}
|
16
|
+
EXPECTED_RETRY_FLAGS_CHUNK_INSERT = {:should_retry => true, :log_prefix => "ChunkInsert"}
|
17
|
+
|
15
18
|
before(:each) do
|
16
19
|
@origin = Lhm::Table.new('foo')
|
17
20
|
@destination = Lhm::Table.new('bar')
|
18
21
|
@migration = Lhm::Migration.new(@origin, @destination)
|
19
22
|
@connection = mock()
|
23
|
+
@connection.stubs(:execute).returns([["dummy"]])
|
20
24
|
# This is a poor man's stub
|
21
25
|
@throttler = Object.new
|
22
26
|
def @throttler.run
|
@@ -38,11 +42,11 @@ describe Lhm::Chunker do
|
|
38
42
|
5
|
39
43
|
end
|
40
44
|
|
41
|
-
@connection.expects(:select_value).with(regexp_matches(/where id >= 1 order by id limit 1 offset 4/),
|
42
|
-
@connection.expects(:select_value).with(regexp_matches(/where id >= 8 order by id limit 1 offset 4/),
|
43
|
-
@connection.expects(:update).with(regexp_matches(/between 1 and 7/),
|
44
|
-
@connection.expects(:update).with(regexp_matches(/between 8 and 10/),
|
45
|
-
@connection.expects(:execute).twice.with(regexp_matches(/show warnings/),
|
45
|
+
@connection.expects(:select_value).with(regexp_matches(/where id >= 1 order by id limit 1 offset 4/),EXPECTED_RETRY_FLAGS_CHUNKER).returns(7)
|
46
|
+
@connection.expects(:select_value).with(regexp_matches(/where id >= 8 order by id limit 1 offset 4/),EXPECTED_RETRY_FLAGS_CHUNKER).returns(21)
|
47
|
+
@connection.expects(:update).with(regexp_matches(/between 1 and 7/),EXPECTED_RETRY_FLAGS_CHUNK_INSERT).returns(2)
|
48
|
+
@connection.expects(:update).with(regexp_matches(/between 8 and 10/),EXPECTED_RETRY_FLAGS_CHUNK_INSERT).returns(2)
|
49
|
+
@connection.expects(:execute).twice.with(regexp_matches(/show warnings/),EXPECTED_RETRY_FLAGS_CHUNKER).returns([])
|
46
50
|
|
47
51
|
@chunker.run
|
48
52
|
end
|
@@ -53,17 +57,17 @@ describe Lhm::Chunker do
|
|
53
57
|
2
|
54
58
|
end
|
55
59
|
|
56
|
-
@connection.expects(:select_value).with(regexp_matches(/where id >= 1 order by id limit 1 offset 1/),
|
57
|
-
@connection.expects(:select_value).with(regexp_matches(/where id >= 3 order by id limit 1 offset 1/),
|
58
|
-
@connection.expects(:select_value).with(regexp_matches(/where id >= 5 order by id limit 1 offset 1/),
|
59
|
-
@connection.expects(:select_value).with(regexp_matches(/where id >= 7 order by id limit 1 offset 1/),
|
60
|
-
@connection.expects(:select_value).with(regexp_matches(/where id >= 9 order by id limit 1 offset 1/),
|
60
|
+
@connection.expects(:select_value).with(regexp_matches(/where id >= 1 order by id limit 1 offset 1/),EXPECTED_RETRY_FLAGS_CHUNKER).returns(2)
|
61
|
+
@connection.expects(:select_value).with(regexp_matches(/where id >= 3 order by id limit 1 offset 1/),EXPECTED_RETRY_FLAGS_CHUNKER).returns(4)
|
62
|
+
@connection.expects(:select_value).with(regexp_matches(/where id >= 5 order by id limit 1 offset 1/),EXPECTED_RETRY_FLAGS_CHUNKER).returns(6)
|
63
|
+
@connection.expects(:select_value).with(regexp_matches(/where id >= 7 order by id limit 1 offset 1/),EXPECTED_RETRY_FLAGS_CHUNKER).returns(8)
|
64
|
+
@connection.expects(:select_value).with(regexp_matches(/where id >= 9 order by id limit 1 offset 1/),EXPECTED_RETRY_FLAGS_CHUNKER).returns(10)
|
61
65
|
|
62
|
-
@connection.expects(:update).with(regexp_matches(/between 1 and 2/),
|
63
|
-
@connection.expects(:update).with(regexp_matches(/between 3 and 4/),
|
64
|
-
@connection.expects(:update).with(regexp_matches(/between 5 and 6/),
|
65
|
-
@connection.expects(:update).with(regexp_matches(/between 7 and 8/),
|
66
|
-
@connection.expects(:update).with(regexp_matches(/between 9 and 10/),
|
66
|
+
@connection.expects(:update).with(regexp_matches(/between 1 and 2/),EXPECTED_RETRY_FLAGS_CHUNK_INSERT).returns(2)
|
67
|
+
@connection.expects(:update).with(regexp_matches(/between 3 and 4/),EXPECTED_RETRY_FLAGS_CHUNK_INSERT).returns(2)
|
68
|
+
@connection.expects(:update).with(regexp_matches(/between 5 and 6/),EXPECTED_RETRY_FLAGS_CHUNK_INSERT).returns(2)
|
69
|
+
@connection.expects(:update).with(regexp_matches(/between 7 and 8/),EXPECTED_RETRY_FLAGS_CHUNK_INSERT).returns(2)
|
70
|
+
@connection.expects(:update).with(regexp_matches(/between 9 and 10/),EXPECTED_RETRY_FLAGS_CHUNK_INSERT).returns(2)
|
67
71
|
|
68
72
|
@chunker.run
|
69
73
|
end
|
@@ -80,17 +84,17 @@ describe Lhm::Chunker do
|
|
80
84
|
end
|
81
85
|
end
|
82
86
|
|
83
|
-
@connection.expects(:select_value).with(regexp_matches(/where id >= 1 order by id limit 1 offset 1/),
|
84
|
-
@connection.expects(:select_value).with(regexp_matches(/where id >= 3 order by id limit 1 offset 2/),
|
85
|
-
@connection.expects(:select_value).with(regexp_matches(/where id >= 6 order by id limit 1 offset 2/),
|
86
|
-
@connection.expects(:select_value).with(regexp_matches(/where id >= 9 order by id limit 1 offset 2/),
|
87
|
+
@connection.expects(:select_value).with(regexp_matches(/where id >= 1 order by id limit 1 offset 1/),EXPECTED_RETRY_FLAGS_CHUNKER).returns(2)
|
88
|
+
@connection.expects(:select_value).with(regexp_matches(/where id >= 3 order by id limit 1 offset 2/),EXPECTED_RETRY_FLAGS_CHUNKER).returns(5)
|
89
|
+
@connection.expects(:select_value).with(regexp_matches(/where id >= 6 order by id limit 1 offset 2/),EXPECTED_RETRY_FLAGS_CHUNKER).returns(8)
|
90
|
+
@connection.expects(:select_value).with(regexp_matches(/where id >= 9 order by id limit 1 offset 2/),EXPECTED_RETRY_FLAGS_CHUNKER).returns(nil)
|
87
91
|
|
88
|
-
@connection.expects(:update).with(regexp_matches(/between 1 and 2/),
|
89
|
-
@connection.expects(:update).with(regexp_matches(/between 3 and 5/),
|
90
|
-
@connection.expects(:update).with(regexp_matches(/between 6 and 8/),
|
91
|
-
@connection.expects(:update).with(regexp_matches(/between 9 and 10/),
|
92
|
+
@connection.expects(:update).with(regexp_matches(/between 1 and 2/),EXPECTED_RETRY_FLAGS_CHUNK_INSERT).returns(2)
|
93
|
+
@connection.expects(:update).with(regexp_matches(/between 3 and 5/),EXPECTED_RETRY_FLAGS_CHUNK_INSERT).returns(2)
|
94
|
+
@connection.expects(:update).with(regexp_matches(/between 6 and 8/),EXPECTED_RETRY_FLAGS_CHUNK_INSERT).returns(2)
|
95
|
+
@connection.expects(:update).with(regexp_matches(/between 9 and 10/),EXPECTED_RETRY_FLAGS_CHUNK_INSERT).returns(2)
|
92
96
|
|
93
|
-
@connection.expects(:execute).twice.with(regexp_matches(/show warnings/),
|
97
|
+
@connection.expects(:execute).twice.with(regexp_matches(/show warnings/),EXPECTED_RETRY_FLAGS_CHUNKER).returns([])
|
94
98
|
|
95
99
|
@chunker.run
|
96
100
|
end
|
@@ -100,8 +104,8 @@ describe Lhm::Chunker do
|
|
100
104
|
:start => 1,
|
101
105
|
:limit => 1)
|
102
106
|
|
103
|
-
@connection.expects(:select_value).with(regexp_matches(/where id >= 1 order by id limit 1 offset 0/),
|
104
|
-
@connection.expects(:update).with(regexp_matches(/between 1 and 1/),
|
107
|
+
@connection.expects(:select_value).with(regexp_matches(/where id >= 1 order by id limit 1 offset 0/),EXPECTED_RETRY_FLAGS_CHUNKER).returns(nil)
|
108
|
+
@connection.expects(:update).with(regexp_matches(/between 1 and 1/),EXPECTED_RETRY_FLAGS_CHUNK_INSERT).returns(1)
|
105
109
|
|
106
110
|
@chunker.run
|
107
111
|
end
|
@@ -114,17 +118,17 @@ describe Lhm::Chunker do
|
|
114
118
|
2
|
115
119
|
end
|
116
120
|
|
117
|
-
@connection.expects(:select_value).with(regexp_matches(/where id >= 2 order by id limit 1 offset 1/),
|
118
|
-
@connection.expects(:select_value).with(regexp_matches(/where id >= 4 order by id limit 1 offset 1/),
|
119
|
-
@connection.expects(:select_value).with(regexp_matches(/where id >= 6 order by id limit 1 offset 1/),
|
120
|
-
@connection.expects(:select_value).with(regexp_matches(/where id >= 8 order by id limit 1 offset 1/),
|
121
|
-
@connection.expects(:select_value).with(regexp_matches(/where id >= 10 order by id limit 1 offset 1/),
|
121
|
+
@connection.expects(:select_value).with(regexp_matches(/where id >= 2 order by id limit 1 offset 1/),EXPECTED_RETRY_FLAGS_CHUNKER).returns(3)
|
122
|
+
@connection.expects(:select_value).with(regexp_matches(/where id >= 4 order by id limit 1 offset 1/),EXPECTED_RETRY_FLAGS_CHUNKER).returns(5)
|
123
|
+
@connection.expects(:select_value).with(regexp_matches(/where id >= 6 order by id limit 1 offset 1/),EXPECTED_RETRY_FLAGS_CHUNKER).returns(7)
|
124
|
+
@connection.expects(:select_value).with(regexp_matches(/where id >= 8 order by id limit 1 offset 1/),EXPECTED_RETRY_FLAGS_CHUNKER).returns(9)
|
125
|
+
@connection.expects(:select_value).with(regexp_matches(/where id >= 10 order by id limit 1 offset 1/),EXPECTED_RETRY_FLAGS_CHUNKER).returns(nil)
|
122
126
|
|
123
|
-
@connection.expects(:update).with(regexp_matches(/between 2 and 3/),
|
124
|
-
@connection.expects(:update).with(regexp_matches(/between 4 and 5/),
|
125
|
-
@connection.expects(:update).with(regexp_matches(/between 6 and 7/),
|
126
|
-
@connection.expects(:update).with(regexp_matches(/between 8 and 9/),
|
127
|
-
@connection.expects(:update).with(regexp_matches(/between 10 and 10/),
|
127
|
+
@connection.expects(:update).with(regexp_matches(/between 2 and 3/),EXPECTED_RETRY_FLAGS_CHUNK_INSERT).returns(2)
|
128
|
+
@connection.expects(:update).with(regexp_matches(/between 4 and 5/),EXPECTED_RETRY_FLAGS_CHUNK_INSERT).returns(2)
|
129
|
+
@connection.expects(:update).with(regexp_matches(/between 6 and 7/),EXPECTED_RETRY_FLAGS_CHUNK_INSERT).returns(2)
|
130
|
+
@connection.expects(:update).with(regexp_matches(/between 8 and 9/),EXPECTED_RETRY_FLAGS_CHUNK_INSERT).returns(2)
|
131
|
+
@connection.expects(:update).with(regexp_matches(/between 10 and 10/),EXPECTED_RETRY_FLAGS_CHUNK_INSERT).returns(1)
|
128
132
|
|
129
133
|
@chunker.run
|
130
134
|
end
|
@@ -138,9 +142,9 @@ describe Lhm::Chunker do
|
|
138
142
|
2
|
139
143
|
end
|
140
144
|
|
141
|
-
@connection.expects(:select_value).with(regexp_matches(/where id >= 1 order by id limit 1 offset 1/),
|
142
|
-
@connection.expects(:update).with(regexp_matches(/where \(foo.created_at > '2013-07-10' or foo.baz = 'quux'\) and `foo`/),
|
143
|
-
@connection.expects(:execute).with(regexp_matches(/show warnings/),
|
145
|
+
@connection.expects(:select_value).with(regexp_matches(/where id >= 1 order by id limit 1 offset 1/),EXPECTED_RETRY_FLAGS_CHUNKER).returns(2)
|
146
|
+
@connection.expects(:update).with(regexp_matches(/where \(foo.created_at > '2013-07-10' or foo.baz = 'quux'\) and `foo`/),EXPECTED_RETRY_FLAGS_CHUNK_INSERT).returns(1)
|
147
|
+
@connection.expects(:execute).with(regexp_matches(/show warnings/),EXPECTED_RETRY_FLAGS_CHUNKER).returns([])
|
144
148
|
|
145
149
|
def @migration.conditions
|
146
150
|
"where foo.created_at > '2013-07-10' or foo.baz = 'quux'"
|
@@ -158,9 +162,9 @@ describe Lhm::Chunker do
|
|
158
162
|
2
|
159
163
|
end
|
160
164
|
|
161
|
-
@connection.expects(:select_value).with(regexp_matches(/where id >= 1 order by id limit 1 offset 1/),
|
162
|
-
@connection.expects(:update).with(regexp_matches(/inner join bar on foo.id = bar.foo_id and/),
|
163
|
-
@connection.expects(:execute).with(regexp_matches(/show warnings/),
|
165
|
+
@connection.expects(:select_value).with(regexp_matches(/where id >= 1 order by id limit 1 offset 1/),EXPECTED_RETRY_FLAGS_CHUNKER).returns(2)
|
166
|
+
@connection.expects(:update).with(regexp_matches(/inner join bar on foo.id = bar.foo_id and/),EXPECTED_RETRY_FLAGS_CHUNK_INSERT).returns(1)
|
167
|
+
@connection.expects(:execute).with(regexp_matches(/show warnings/),EXPECTED_RETRY_FLAGS_CHUNKER).returns([])
|
164
168
|
|
165
169
|
def @migration.conditions
|
166
170
|
'inner join bar on foo.id = bar.foo_id'
|
@@ -1,4 +1,5 @@
|
|
1
1
|
require 'lhm/connection'
|
2
|
+
require 'lhm/proxysql_helper'
|
2
3
|
|
3
4
|
describe Lhm::Connection do
|
4
5
|
|
@@ -12,10 +13,15 @@ describe Lhm::Connection do
|
|
12
13
|
it "Should find use calling file as prefix" do
|
13
14
|
ar_connection = mock()
|
14
15
|
ar_connection.stubs(:execute).raises(LOCK_WAIT).then.returns(true)
|
16
|
+
ar_connection.stubs(:active?).returns(true)
|
15
17
|
|
16
|
-
connection = Lhm::Connection.new(connection: ar_connection
|
18
|
+
connection = Lhm::Connection.new(connection: ar_connection, options: {
|
19
|
+
retriable: {
|
20
|
+
base_interval: 0
|
21
|
+
}
|
22
|
+
})
|
17
23
|
|
18
|
-
connection.execute("SHOW TABLES",
|
24
|
+
connection.execute("SHOW TABLES", should_retry: true)
|
19
25
|
|
20
26
|
log_messages = @logs.string.split("\n")
|
21
27
|
assert_equal(1, log_messages.length)
|
@@ -27,10 +33,16 @@ describe Lhm::Connection do
|
|
27
33
|
ar_connection.stubs(:execute).raises(LOCK_WAIT)
|
28
34
|
.then.raises(LOCK_WAIT)
|
29
35
|
.then.returns(true)
|
36
|
+
ar_connection.stubs(:active?).returns(true)
|
30
37
|
|
31
|
-
connection = Lhm::Connection.new(connection: ar_connection
|
38
|
+
connection = Lhm::Connection.new(connection: ar_connection, options: {
|
39
|
+
retriable: {
|
40
|
+
base_interval: 0,
|
41
|
+
tries: 3
|
42
|
+
}
|
43
|
+
})
|
32
44
|
|
33
|
-
connection.execute("SHOW TABLES",
|
45
|
+
connection.execute("SHOW TABLES", should_retry: true)
|
34
46
|
|
35
47
|
log_messages = @logs.string.split("\n")
|
36
48
|
assert_equal(2, log_messages.length)
|
@@ -41,10 +53,16 @@ describe Lhm::Connection do
|
|
41
53
|
ar_connection.stubs(:update).raises(LOCK_WAIT)
|
42
54
|
.then.raises(LOCK_WAIT)
|
43
55
|
.then.returns(1)
|
56
|
+
ar_connection.stubs(:active?).returns(true)
|
44
57
|
|
45
|
-
connection = Lhm::Connection.new(connection: ar_connection
|
58
|
+
connection = Lhm::Connection.new(connection: ar_connection, options: {
|
59
|
+
retriable: {
|
60
|
+
base_interval: 0,
|
61
|
+
tries: 3
|
62
|
+
}
|
63
|
+
})
|
46
64
|
|
47
|
-
val = connection.update("SHOW TABLES",
|
65
|
+
val = connection.update("SHOW TABLES", should_retry: true)
|
48
66
|
|
49
67
|
log_messages = @logs.string.split("\n")
|
50
68
|
assert_equal val, 1
|
@@ -56,13 +74,38 @@ describe Lhm::Connection do
|
|
56
74
|
ar_connection.stubs(:select_value).raises(LOCK_WAIT)
|
57
75
|
.then.raises(LOCK_WAIT)
|
58
76
|
.then.returns("dummy")
|
77
|
+
ar_connection.stubs(:active?).returns(true)
|
59
78
|
|
60
|
-
connection = Lhm::Connection.new(connection: ar_connection
|
79
|
+
connection = Lhm::Connection.new(connection: ar_connection, options: {
|
80
|
+
retriable: {
|
81
|
+
base_interval: 0,
|
82
|
+
tries: 3
|
83
|
+
}
|
84
|
+
})
|
61
85
|
|
62
|
-
val = connection.select_value("SHOW TABLES",
|
86
|
+
val = connection.select_value("SHOW TABLES", should_retry: true)
|
63
87
|
|
64
88
|
log_messages = @logs.string.split("\n")
|
65
89
|
assert_equal val, "dummy"
|
66
90
|
assert_equal(2, log_messages.length)
|
67
91
|
end
|
92
|
+
|
93
|
+
it "Queries should be tagged with ProxySQL tag if reconnect_with_consistent_host is enabled" do
|
94
|
+
ar_connection = mock()
|
95
|
+
ar_connection.expects(:public_send).with(:select_value, "SHOW TABLES #{Lhm::ProxySQLHelper::ANNOTATION}").returns("dummy")
|
96
|
+
ar_connection.stubs(:execute).times(4).returns([["dummy"]])
|
97
|
+
ar_connection.stubs(:active?).returns(true)
|
98
|
+
|
99
|
+
connection = Lhm::Connection.new(connection: ar_connection, options: {
|
100
|
+
reconnect_with_consistent_host: true,
|
101
|
+
retriable: {
|
102
|
+
base_interval: 0,
|
103
|
+
tries: 3
|
104
|
+
}
|
105
|
+
})
|
106
|
+
|
107
|
+
val = connection.select_value("SHOW TABLES", should_retry: true)
|
108
|
+
|
109
|
+
assert_equal val, "dummy"
|
110
|
+
end
|
68
111
|
end
|
data/spec/unit/entangler_spec.rb
CHANGED
@@ -63,42 +63,94 @@ describe Lhm::Entangler do
|
|
63
63
|
it 'should retry trigger creation when it hits a lock wait timeout' do
|
64
64
|
tries = 1
|
65
65
|
ar_connection = mock()
|
66
|
-
ar_connection.
|
67
|
-
|
68
|
-
|
69
|
-
|
70
|
-
|
66
|
+
ar_connection.stubs(:execute)
|
67
|
+
.returns([["dummy"]], [["dummy"]], [["dummy"]])
|
68
|
+
.then
|
69
|
+
.raises(Mysql2::Error, 'Lock wait timeout exceeded; try restarting transaction')
|
70
|
+
ar_connection.stubs(:active?).returns(true)
|
71
|
+
|
72
|
+
connection = Lhm::Connection.new(connection: ar_connection, options: {
|
73
|
+
reconnect_with_consistent_host: true,
|
74
|
+
retriable: {
|
75
|
+
base_interval: 0,
|
76
|
+
tries: tries
|
77
|
+
}
|
78
|
+
})
|
79
|
+
|
80
|
+
@entangler = Lhm::Entangler.new(@migration, connection)
|
71
81
|
|
72
82
|
assert_raises(Mysql2::Error) { @entangler.before }
|
73
83
|
end
|
74
84
|
|
75
85
|
it 'should not retry trigger creation with other mysql errors' do
|
76
86
|
ar_connection = mock()
|
77
|
-
ar_connection.
|
78
|
-
|
79
|
-
|
80
|
-
|
87
|
+
ar_connection.stubs(:execute)
|
88
|
+
.returns([["dummy"]], [["dummy"]], [["dummy"]])
|
89
|
+
.then
|
90
|
+
.raises(Mysql2::Error, 'The MySQL server is running with the --read-only option so it cannot execute this statement.')
|
91
|
+
ar_connection.stubs(:active?).returns(true)
|
92
|
+
connection = Lhm::Connection.new(connection: ar_connection, options: {
|
93
|
+
reconnect_with_consistent_host: true,
|
94
|
+
retriable: {
|
95
|
+
base_interval: 0
|
96
|
+
},
|
97
|
+
})
|
98
|
+
|
99
|
+
@entangler = Lhm::Entangler.new(@migration, connection)
|
81
100
|
assert_raises(Mysql2::Error) { @entangler.before }
|
82
101
|
end
|
83
102
|
|
84
103
|
it 'should succesfully finish after retrying' do
|
85
104
|
ar_connection = mock()
|
86
|
-
ar_connection.stubs(:execute)
|
87
|
-
|
88
|
-
|
89
|
-
|
90
|
-
|
105
|
+
ar_connection.stubs(:execute)
|
106
|
+
.returns([["dummy"]], [["dummy"]], [["dummy"]])
|
107
|
+
.then
|
108
|
+
.raises(Mysql2::Error, 'Lock wait timeout exceeded; try restarting transaction')
|
109
|
+
.then
|
110
|
+
.returns([["dummy"]])
|
111
|
+
ar_connection.stubs(:active?).returns(true)
|
112
|
+
|
113
|
+
connection = Lhm::Connection.new(connection: ar_connection, options: {
|
114
|
+
reconnect_with_consistent_host: true,
|
115
|
+
retriable: {
|
116
|
+
base_interval: 0
|
117
|
+
},
|
118
|
+
})
|
119
|
+
|
120
|
+
@entangler = Lhm::Entangler.new(@migration, connection)
|
91
121
|
|
92
122
|
assert @entangler.before
|
93
123
|
end
|
94
124
|
|
95
125
|
it 'should retry as many times as specified by configuration' do
|
96
126
|
ar_connection = mock()
|
97
|
-
ar_connection.
|
98
|
-
|
99
|
-
|
100
|
-
|
101
|
-
|
127
|
+
ar_connection.stubs(:execute)
|
128
|
+
.returns([["dummy"]], [["dummy"]], [["dummy"]]) # initial
|
129
|
+
.then
|
130
|
+
.raises(Mysql2::Error, 'Lock wait timeout exceeded; try restarting transaction')
|
131
|
+
.then
|
132
|
+
.returns([["dummy"]]) # reconnect 1
|
133
|
+
.then
|
134
|
+
.raises(Mysql2::Error, 'Lock wait timeout exceeded; try restarting transaction')
|
135
|
+
.then
|
136
|
+
.returns([["dummy"]]) # reconnect 2
|
137
|
+
.then
|
138
|
+
.raises(Mysql2::Error, 'Lock wait timeout exceeded; try restarting transaction')
|
139
|
+
.then
|
140
|
+
.returns([["dummy"]]) # reconnect 3
|
141
|
+
.then
|
142
|
+
.raises(Mysql2::Error, 'Lock wait timeout exceeded; try restarting transaction') # final error
|
143
|
+
ar_connection.stubs(:active?).returns(true)
|
144
|
+
|
145
|
+
connection = Lhm::Connection.new(connection: ar_connection, options: {
|
146
|
+
reconnect_with_consistent_host: true,
|
147
|
+
retriable: {
|
148
|
+
tries: 5,
|
149
|
+
base_interval: 0
|
150
|
+
},
|
151
|
+
})
|
152
|
+
|
153
|
+
@entangler = Lhm::Entangler.new(@migration, connection)
|
102
154
|
|
103
155
|
assert_raises(Mysql2::Error) { @entangler.before }
|
104
156
|
end
|
data/spec/unit/lhm_spec.rb
CHANGED
@@ -26,4 +26,21 @@ describe Lhm do
|
|
26
26
|
value(Lhm.logger.instance_eval { @logdev }.dev.path).must_equal 'omg.ponies'
|
27
27
|
end
|
28
28
|
end
|
29
|
+
|
30
|
+
describe 'api' do
|
31
|
+
|
32
|
+
before(:each) do
|
33
|
+
@connection = mock()
|
34
|
+
end
|
35
|
+
|
36
|
+
it 'should create a new connection when calling setup' do
|
37
|
+
Lhm.setup(@connection)
|
38
|
+
value(Lhm.connection).must_be_kind_of(Lhm::Connection)
|
39
|
+
end
|
40
|
+
|
41
|
+
it 'should create a new connection when none is created' do
|
42
|
+
ActiveRecord::Base.stubs(:connection).returns(@connection)
|
43
|
+
value(Lhm.connection).must_be_kind_of(Lhm::Connection)
|
44
|
+
end
|
45
|
+
end
|
29
46
|
end
|
@@ -39,7 +39,7 @@ describe Lhm::Throttler::Slave do
|
|
39
39
|
@logs = StringIO.new
|
40
40
|
Lhm.logger = Logger.new(@logs)
|
41
41
|
|
42
|
-
@dummy_mysql_client_config = lambda { {'username' => 'user', 'password' => 'pw', 'database' => 'db'} }
|
42
|
+
@dummy_mysql_client_config = lambda { { 'username' => 'user', 'password' => 'pw', 'database' => 'db' } }
|
43
43
|
end
|
44
44
|
|
45
45
|
describe "#client" do
|
@@ -64,7 +64,7 @@ describe Lhm::Throttler::Slave do
|
|
64
64
|
|
65
65
|
describe 'with proper config' do
|
66
66
|
it "creates a new Mysql2::Client" do
|
67
|
-
expected_config = {username: 'user', password: 'pw', database: 'db', host: 'slave'}
|
67
|
+
expected_config = { username: 'user', password: 'pw', database: 'db', host: 'slave' }
|
68
68
|
Mysql2::Client.stubs(:new).with(expected_config).returns(mock())
|
69
69
|
|
70
70
|
assert Lhm::Throttler::Slave.new('slave', @dummy_mysql_client_config).connection
|
@@ -73,8 +73,12 @@ describe Lhm::Throttler::Slave do
|
|
73
73
|
|
74
74
|
describe 'with active record config' do
|
75
75
|
it 'logs and creates client' do
|
76
|
-
active_record_config = {username: 'user', password: 'pw', database: 'db'}
|
77
|
-
ActiveRecord::
|
76
|
+
active_record_config = { username: 'user', password: 'pw', database: 'db' }
|
77
|
+
if ActiveRecord::VERSION::MAJOR > 6 || ActiveRecord::VERSION::MAJOR == 6 && ActiveRecord::VERSION::MINOR >= 1
|
78
|
+
ActiveRecord::Base.stubs(:connection_pool).returns(stub(db_config: stub(configuration_hash: active_record_config)))
|
79
|
+
else
|
80
|
+
ActiveRecord::Base.stubs(:connection_pool).returns(stub(spec: stub(config: active_record_config)))
|
81
|
+
end
|
78
82
|
|
79
83
|
Mysql2::Client.stubs(:new).returns(mock())
|
80
84
|
|
@@ -92,9 +96,9 @@ describe Lhm::Throttler::Slave do
|
|
92
96
|
class Connection
|
93
97
|
def self.query(query)
|
94
98
|
if query == Lhm::Throttler::Slave::SQL_SELECT_MAX_SLAVE_LAG
|
95
|
-
[{'Seconds_Behind_Master' => 20}]
|
99
|
+
[{ 'Seconds_Behind_Master' => 20 }]
|
96
100
|
elsif query == Lhm::Throttler::Slave::SQL_SELECT_SLAVE_HOSTS
|
97
|
-
[{'host' => '1.1.1.1:80'}]
|
101
|
+
[{ 'host' => '1.1.1.1:80' }]
|
98
102
|
end
|
99
103
|
end
|
100
104
|
end
|
@@ -104,7 +108,7 @@ describe Lhm::Throttler::Slave do
|
|
104
108
|
|
105
109
|
class StoppedConnection
|
106
110
|
def self.query(query)
|
107
|
-
[{'Seconds_Behind_Master' => nil}]
|
111
|
+
[{ 'Seconds_Behind_Master' => nil }]
|
108
112
|
end
|
109
113
|
end
|
110
114
|
|
@@ -138,7 +142,7 @@ describe Lhm::Throttler::Slave do
|
|
138
142
|
Lhm::Throttler::Slave.any_instance.stubs(:config).returns([])
|
139
143
|
|
140
144
|
slave = Lhm::Throttler::Slave.new('slave', @dummy_mysql_client_config)
|
141
|
-
|
145
|
+
Logger.any_instance.expects(:info).with("Unable to connect and/or query slave: Can't connect to MySQL server")
|
142
146
|
assert_equal(0, slave.lag)
|
143
147
|
end
|
144
148
|
end
|
@@ -286,7 +290,7 @@ describe Lhm::Throttler::SlaveLag do
|
|
286
290
|
describe 'with the :check_only option' do
|
287
291
|
describe 'with a callable argument' do
|
288
292
|
before do
|
289
|
-
check_only = lambda {{'host' => '1.1.1.3'}}
|
293
|
+
check_only = lambda { { 'host' => '1.1.1.3' } }
|
290
294
|
@throttler = Lhm::Throttler::SlaveLag.new :check_only => check_only
|
291
295
|
end
|
292
296
|
|
@@ -300,6 +304,7 @@ describe Lhm::Throttler::SlaveLag do
|
|
300
304
|
describe 'with a non-callable argument' do
|
301
305
|
before do
|
302
306
|
@throttler = Lhm::Throttler::SlaveLag.new :check_only => 'I cannot be called'
|
307
|
+
|
303
308
|
def @throttler.master_slave_hosts
|
304
309
|
['1.1.1.1', '1.1.1.4']
|
305
310
|
end
|