lhm-shopify 3.5.0 → 3.5.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.github/workflows/test.yml +17 -4
- data/.gitignore +0 -2
- data/Appraisals +24 -0
- data/CHANGELOG.md +23 -0
- data/Gemfile.lock +66 -0
- data/README.md +53 -0
- data/Rakefile +6 -5
- data/dev.yml +18 -3
- data/docker-compose.yml +15 -3
- data/gemfiles/activerecord_5.2.gemfile +9 -0
- data/gemfiles/activerecord_5.2.gemfile.lock +65 -0
- data/gemfiles/activerecord_6.0.gemfile +7 -0
- data/gemfiles/activerecord_6.0.gemfile.lock +67 -0
- data/gemfiles/activerecord_6.1.gemfile +7 -0
- data/gemfiles/activerecord_6.1.gemfile.lock +66 -0
- data/gemfiles/activerecord_7.0.0.alpha2.gemfile +7 -0
- data/gemfiles/activerecord_7.0.0.alpha2.gemfile.lock +64 -0
- data/lhm.gemspec +7 -3
- data/lib/lhm/atomic_switcher.rb +4 -3
- data/lib/lhm/chunk_insert.rb +7 -3
- data/lib/lhm/chunker.rb +6 -6
- data/lib/lhm/cleanup/current.rb +4 -1
- data/lib/lhm/connection.rb +66 -19
- data/lib/lhm/entangler.rb +5 -4
- data/lib/lhm/invoker.rb +5 -3
- data/lib/lhm/locked_switcher.rb +2 -0
- data/lib/lhm/proxysql_helper.rb +10 -0
- data/lib/lhm/sql_retry.rb +135 -11
- data/lib/lhm/throttler/slave_lag.rb +19 -2
- data/lib/lhm/version.rb +1 -1
- data/lib/lhm.rb +32 -12
- data/scripts/mysql/writer/create_users.sql +3 -0
- data/spec/integration/atomic_switcher_spec.rb +38 -10
- data/spec/integration/chunk_insert_spec.rb +2 -1
- data/spec/integration/chunker_spec.rb +8 -6
- data/spec/integration/database.yml +10 -0
- data/spec/integration/entangler_spec.rb +3 -1
- data/spec/integration/integration_helper.rb +20 -4
- data/spec/integration/lhm_spec.rb +75 -0
- data/spec/integration/proxysql_spec.rb +34 -0
- data/spec/integration/sql_retry/db_connection_helper.rb +52 -0
- data/spec/integration/sql_retry/lock_wait_spec.rb +8 -6
- data/spec/integration/sql_retry/lock_wait_timeout_test_helper.rb +19 -9
- data/spec/integration/sql_retry/proxysql_helper.rb +22 -0
- data/spec/integration/sql_retry/retry_with_proxysql_spec.rb +108 -0
- data/spec/integration/toxiproxy_helper.rb +40 -0
- data/spec/test_helper.rb +21 -0
- data/spec/unit/chunk_insert_spec.rb +7 -2
- data/spec/unit/chunker_spec.rb +46 -42
- data/spec/unit/connection_spec.rb +51 -8
- data/spec/unit/entangler_spec.rb +71 -19
- data/spec/unit/lhm_spec.rb +17 -0
- data/spec/unit/throttler/slave_lag_spec.rb +14 -9
- metadata +76 -11
- data/gemfiles/ar-2.3_mysql.gemfile +0 -6
- data/gemfiles/ar-3.2_mysql.gemfile +0 -5
- data/gemfiles/ar-3.2_mysql2.gemfile +0 -5
- data/gemfiles/ar-4.0_mysql2.gemfile +0 -5
- data/gemfiles/ar-4.1_mysql2.gemfile +0 -5
- data/gemfiles/ar-4.2_mysql2.gemfile +0 -5
- data/gemfiles/ar-5.0_mysql2.gemfile +0 -5
data/spec/test_helper.rb
CHANGED
@@ -10,6 +10,8 @@ require 'minitest/autorun'
|
|
10
10
|
require 'minitest/spec'
|
11
11
|
require 'minitest/mock'
|
12
12
|
require 'mocha/minitest'
|
13
|
+
require 'after_do'
|
14
|
+
require 'byebug'
|
13
15
|
require 'pathname'
|
14
16
|
require 'lhm'
|
15
17
|
|
@@ -17,6 +19,8 @@ $project = Pathname.new(File.dirname(__FILE__) + '/..').cleanpath
|
|
17
19
|
$spec = $project.join('spec')
|
18
20
|
$fixtures = $spec.join('fixtures')
|
19
21
|
|
22
|
+
$db_name = 'test'
|
23
|
+
|
20
24
|
require 'active_record'
|
21
25
|
require 'mysql2'
|
22
26
|
|
@@ -43,3 +47,20 @@ end
|
|
43
47
|
def throttler
|
44
48
|
Lhm::Throttler::Time.new(:stride => 100)
|
45
49
|
end
|
50
|
+
|
51
|
+
def init_test_db
|
52
|
+
db_config = YAML.load_file(File.expand_path(File.dirname(__FILE__)) + '/integration/database.yml')
|
53
|
+
conn = Mysql2::Client.new(
|
54
|
+
:host => '127.0.0.1',
|
55
|
+
:username => db_config['master']['user'],
|
56
|
+
:password => db_config['master']['password'],
|
57
|
+
:port => db_config['master']['port']
|
58
|
+
)
|
59
|
+
|
60
|
+
conn.query("DROP DATABASE IF EXISTS #{$db_name}")
|
61
|
+
conn.query("CREATE DATABASE #{$db_name}")
|
62
|
+
end
|
63
|
+
|
64
|
+
init_test_db
|
65
|
+
|
66
|
+
|
@@ -4,17 +4,22 @@
|
|
4
4
|
require File.expand_path(File.dirname(__FILE__)) + '/unit_helper'
|
5
5
|
|
6
6
|
require 'lhm/chunk_insert'
|
7
|
+
require 'lhm/connection'
|
7
8
|
|
8
9
|
describe Lhm::ChunkInsert do
|
9
10
|
before(:each) do
|
10
|
-
|
11
|
+
ar_connection = mock()
|
12
|
+
ar_connection.stubs(:execute).returns([["dummy"]])
|
13
|
+
@connection = Lhm::Connection.new(connection: ar_connection, options: {reconnect_with_consistent_host: false})
|
11
14
|
@origin = Lhm::Table.new('foo')
|
12
15
|
@destination = Lhm::Table.new('bar')
|
13
16
|
end
|
14
17
|
|
15
18
|
describe "#sql" do
|
16
19
|
describe "when migration has no conditions" do
|
17
|
-
before
|
20
|
+
before do
|
21
|
+
@migration = Lhm::Migration.new(@origin, @destination)
|
22
|
+
end
|
18
23
|
|
19
24
|
it "uses a simple where clause" do
|
20
25
|
assert_equal(
|
data/spec/unit/chunker_spec.rb
CHANGED
@@ -12,11 +12,15 @@ require 'lhm/connection'
|
|
12
12
|
describe Lhm::Chunker do
|
13
13
|
include UnitHelper
|
14
14
|
|
15
|
+
EXPECTED_RETRY_FLAGS_CHUNKER = {:should_retry => true, :log_prefix => "Chunker"}
|
16
|
+
EXPECTED_RETRY_FLAGS_CHUNK_INSERT = {:should_retry => true, :log_prefix => "ChunkInsert"}
|
17
|
+
|
15
18
|
before(:each) do
|
16
19
|
@origin = Lhm::Table.new('foo')
|
17
20
|
@destination = Lhm::Table.new('bar')
|
18
21
|
@migration = Lhm::Migration.new(@origin, @destination)
|
19
22
|
@connection = mock()
|
23
|
+
@connection.stubs(:execute).returns([["dummy"]])
|
20
24
|
# This is a poor man's stub
|
21
25
|
@throttler = Object.new
|
22
26
|
def @throttler.run
|
@@ -38,11 +42,11 @@ describe Lhm::Chunker do
|
|
38
42
|
5
|
39
43
|
end
|
40
44
|
|
41
|
-
@connection.expects(:select_value).with(regexp_matches(/where id >= 1 order by id limit 1 offset 4/),
|
42
|
-
@connection.expects(:select_value).with(regexp_matches(/where id >= 8 order by id limit 1 offset 4/),
|
43
|
-
@connection.expects(:update).with(regexp_matches(/between 1 and 7/),
|
44
|
-
@connection.expects(:update).with(regexp_matches(/between 8 and 10/),
|
45
|
-
@connection.expects(:execute).twice.with(regexp_matches(/show warnings/),
|
45
|
+
@connection.expects(:select_value).with(regexp_matches(/where id >= 1 order by id limit 1 offset 4/),EXPECTED_RETRY_FLAGS_CHUNKER).returns(7)
|
46
|
+
@connection.expects(:select_value).with(regexp_matches(/where id >= 8 order by id limit 1 offset 4/),EXPECTED_RETRY_FLAGS_CHUNKER).returns(21)
|
47
|
+
@connection.expects(:update).with(regexp_matches(/between 1 and 7/),EXPECTED_RETRY_FLAGS_CHUNK_INSERT).returns(2)
|
48
|
+
@connection.expects(:update).with(regexp_matches(/between 8 and 10/),EXPECTED_RETRY_FLAGS_CHUNK_INSERT).returns(2)
|
49
|
+
@connection.expects(:execute).twice.with(regexp_matches(/show warnings/),EXPECTED_RETRY_FLAGS_CHUNKER).returns([])
|
46
50
|
|
47
51
|
@chunker.run
|
48
52
|
end
|
@@ -53,17 +57,17 @@ describe Lhm::Chunker do
|
|
53
57
|
2
|
54
58
|
end
|
55
59
|
|
56
|
-
@connection.expects(:select_value).with(regexp_matches(/where id >= 1 order by id limit 1 offset 1/),
|
57
|
-
@connection.expects(:select_value).with(regexp_matches(/where id >= 3 order by id limit 1 offset 1/),
|
58
|
-
@connection.expects(:select_value).with(regexp_matches(/where id >= 5 order by id limit 1 offset 1/),
|
59
|
-
@connection.expects(:select_value).with(regexp_matches(/where id >= 7 order by id limit 1 offset 1/),
|
60
|
-
@connection.expects(:select_value).with(regexp_matches(/where id >= 9 order by id limit 1 offset 1/),
|
60
|
+
@connection.expects(:select_value).with(regexp_matches(/where id >= 1 order by id limit 1 offset 1/),EXPECTED_RETRY_FLAGS_CHUNKER).returns(2)
|
61
|
+
@connection.expects(:select_value).with(regexp_matches(/where id >= 3 order by id limit 1 offset 1/),EXPECTED_RETRY_FLAGS_CHUNKER).returns(4)
|
62
|
+
@connection.expects(:select_value).with(regexp_matches(/where id >= 5 order by id limit 1 offset 1/),EXPECTED_RETRY_FLAGS_CHUNKER).returns(6)
|
63
|
+
@connection.expects(:select_value).with(regexp_matches(/where id >= 7 order by id limit 1 offset 1/),EXPECTED_RETRY_FLAGS_CHUNKER).returns(8)
|
64
|
+
@connection.expects(:select_value).with(regexp_matches(/where id >= 9 order by id limit 1 offset 1/),EXPECTED_RETRY_FLAGS_CHUNKER).returns(10)
|
61
65
|
|
62
|
-
@connection.expects(:update).with(regexp_matches(/between 1 and 2/),
|
63
|
-
@connection.expects(:update).with(regexp_matches(/between 3 and 4/),
|
64
|
-
@connection.expects(:update).with(regexp_matches(/between 5 and 6/),
|
65
|
-
@connection.expects(:update).with(regexp_matches(/between 7 and 8/),
|
66
|
-
@connection.expects(:update).with(regexp_matches(/between 9 and 10/),
|
66
|
+
@connection.expects(:update).with(regexp_matches(/between 1 and 2/),EXPECTED_RETRY_FLAGS_CHUNK_INSERT).returns(2)
|
67
|
+
@connection.expects(:update).with(regexp_matches(/between 3 and 4/),EXPECTED_RETRY_FLAGS_CHUNK_INSERT).returns(2)
|
68
|
+
@connection.expects(:update).with(regexp_matches(/between 5 and 6/),EXPECTED_RETRY_FLAGS_CHUNK_INSERT).returns(2)
|
69
|
+
@connection.expects(:update).with(regexp_matches(/between 7 and 8/),EXPECTED_RETRY_FLAGS_CHUNK_INSERT).returns(2)
|
70
|
+
@connection.expects(:update).with(regexp_matches(/between 9 and 10/),EXPECTED_RETRY_FLAGS_CHUNK_INSERT).returns(2)
|
67
71
|
|
68
72
|
@chunker.run
|
69
73
|
end
|
@@ -80,17 +84,17 @@ describe Lhm::Chunker do
|
|
80
84
|
end
|
81
85
|
end
|
82
86
|
|
83
|
-
@connection.expects(:select_value).with(regexp_matches(/where id >= 1 order by id limit 1 offset 1/),
|
84
|
-
@connection.expects(:select_value).with(regexp_matches(/where id >= 3 order by id limit 1 offset 2/),
|
85
|
-
@connection.expects(:select_value).with(regexp_matches(/where id >= 6 order by id limit 1 offset 2/),
|
86
|
-
@connection.expects(:select_value).with(regexp_matches(/where id >= 9 order by id limit 1 offset 2/),
|
87
|
+
@connection.expects(:select_value).with(regexp_matches(/where id >= 1 order by id limit 1 offset 1/),EXPECTED_RETRY_FLAGS_CHUNKER).returns(2)
|
88
|
+
@connection.expects(:select_value).with(regexp_matches(/where id >= 3 order by id limit 1 offset 2/),EXPECTED_RETRY_FLAGS_CHUNKER).returns(5)
|
89
|
+
@connection.expects(:select_value).with(regexp_matches(/where id >= 6 order by id limit 1 offset 2/),EXPECTED_RETRY_FLAGS_CHUNKER).returns(8)
|
90
|
+
@connection.expects(:select_value).with(regexp_matches(/where id >= 9 order by id limit 1 offset 2/),EXPECTED_RETRY_FLAGS_CHUNKER).returns(nil)
|
87
91
|
|
88
|
-
@connection.expects(:update).with(regexp_matches(/between 1 and 2/),
|
89
|
-
@connection.expects(:update).with(regexp_matches(/between 3 and 5/),
|
90
|
-
@connection.expects(:update).with(regexp_matches(/between 6 and 8/),
|
91
|
-
@connection.expects(:update).with(regexp_matches(/between 9 and 10/),
|
92
|
+
@connection.expects(:update).with(regexp_matches(/between 1 and 2/),EXPECTED_RETRY_FLAGS_CHUNK_INSERT).returns(2)
|
93
|
+
@connection.expects(:update).with(regexp_matches(/between 3 and 5/),EXPECTED_RETRY_FLAGS_CHUNK_INSERT).returns(2)
|
94
|
+
@connection.expects(:update).with(regexp_matches(/between 6 and 8/),EXPECTED_RETRY_FLAGS_CHUNK_INSERT).returns(2)
|
95
|
+
@connection.expects(:update).with(regexp_matches(/between 9 and 10/),EXPECTED_RETRY_FLAGS_CHUNK_INSERT).returns(2)
|
92
96
|
|
93
|
-
@connection.expects(:execute).twice.with(regexp_matches(/show warnings/),
|
97
|
+
@connection.expects(:execute).twice.with(regexp_matches(/show warnings/),EXPECTED_RETRY_FLAGS_CHUNKER).returns([])
|
94
98
|
|
95
99
|
@chunker.run
|
96
100
|
end
|
@@ -100,8 +104,8 @@ describe Lhm::Chunker do
|
|
100
104
|
:start => 1,
|
101
105
|
:limit => 1)
|
102
106
|
|
103
|
-
@connection.expects(:select_value).with(regexp_matches(/where id >= 1 order by id limit 1 offset 0/),
|
104
|
-
@connection.expects(:update).with(regexp_matches(/between 1 and 1/),
|
107
|
+
@connection.expects(:select_value).with(regexp_matches(/where id >= 1 order by id limit 1 offset 0/),EXPECTED_RETRY_FLAGS_CHUNKER).returns(nil)
|
108
|
+
@connection.expects(:update).with(regexp_matches(/between 1 and 1/),EXPECTED_RETRY_FLAGS_CHUNK_INSERT).returns(1)
|
105
109
|
|
106
110
|
@chunker.run
|
107
111
|
end
|
@@ -114,17 +118,17 @@ describe Lhm::Chunker do
|
|
114
118
|
2
|
115
119
|
end
|
116
120
|
|
117
|
-
@connection.expects(:select_value).with(regexp_matches(/where id >= 2 order by id limit 1 offset 1/),
|
118
|
-
@connection.expects(:select_value).with(regexp_matches(/where id >= 4 order by id limit 1 offset 1/),
|
119
|
-
@connection.expects(:select_value).with(regexp_matches(/where id >= 6 order by id limit 1 offset 1/),
|
120
|
-
@connection.expects(:select_value).with(regexp_matches(/where id >= 8 order by id limit 1 offset 1/),
|
121
|
-
@connection.expects(:select_value).with(regexp_matches(/where id >= 10 order by id limit 1 offset 1/),
|
121
|
+
@connection.expects(:select_value).with(regexp_matches(/where id >= 2 order by id limit 1 offset 1/),EXPECTED_RETRY_FLAGS_CHUNKER).returns(3)
|
122
|
+
@connection.expects(:select_value).with(regexp_matches(/where id >= 4 order by id limit 1 offset 1/),EXPECTED_RETRY_FLAGS_CHUNKER).returns(5)
|
123
|
+
@connection.expects(:select_value).with(regexp_matches(/where id >= 6 order by id limit 1 offset 1/),EXPECTED_RETRY_FLAGS_CHUNKER).returns(7)
|
124
|
+
@connection.expects(:select_value).with(regexp_matches(/where id >= 8 order by id limit 1 offset 1/),EXPECTED_RETRY_FLAGS_CHUNKER).returns(9)
|
125
|
+
@connection.expects(:select_value).with(regexp_matches(/where id >= 10 order by id limit 1 offset 1/),EXPECTED_RETRY_FLAGS_CHUNKER).returns(nil)
|
122
126
|
|
123
|
-
@connection.expects(:update).with(regexp_matches(/between 2 and 3/),
|
124
|
-
@connection.expects(:update).with(regexp_matches(/between 4 and 5/),
|
125
|
-
@connection.expects(:update).with(regexp_matches(/between 6 and 7/),
|
126
|
-
@connection.expects(:update).with(regexp_matches(/between 8 and 9/),
|
127
|
-
@connection.expects(:update).with(regexp_matches(/between 10 and 10/),
|
127
|
+
@connection.expects(:update).with(regexp_matches(/between 2 and 3/),EXPECTED_RETRY_FLAGS_CHUNK_INSERT).returns(2)
|
128
|
+
@connection.expects(:update).with(regexp_matches(/between 4 and 5/),EXPECTED_RETRY_FLAGS_CHUNK_INSERT).returns(2)
|
129
|
+
@connection.expects(:update).with(regexp_matches(/between 6 and 7/),EXPECTED_RETRY_FLAGS_CHUNK_INSERT).returns(2)
|
130
|
+
@connection.expects(:update).with(regexp_matches(/between 8 and 9/),EXPECTED_RETRY_FLAGS_CHUNK_INSERT).returns(2)
|
131
|
+
@connection.expects(:update).with(regexp_matches(/between 10 and 10/),EXPECTED_RETRY_FLAGS_CHUNK_INSERT).returns(1)
|
128
132
|
|
129
133
|
@chunker.run
|
130
134
|
end
|
@@ -138,9 +142,9 @@ describe Lhm::Chunker do
|
|
138
142
|
2
|
139
143
|
end
|
140
144
|
|
141
|
-
@connection.expects(:select_value).with(regexp_matches(/where id >= 1 order by id limit 1 offset 1/),
|
142
|
-
@connection.expects(:update).with(regexp_matches(/where \(foo.created_at > '2013-07-10' or foo.baz = 'quux'\) and `foo`/),
|
143
|
-
@connection.expects(:execute).with(regexp_matches(/show warnings/),
|
145
|
+
@connection.expects(:select_value).with(regexp_matches(/where id >= 1 order by id limit 1 offset 1/),EXPECTED_RETRY_FLAGS_CHUNKER).returns(2)
|
146
|
+
@connection.expects(:update).with(regexp_matches(/where \(foo.created_at > '2013-07-10' or foo.baz = 'quux'\) and `foo`/),EXPECTED_RETRY_FLAGS_CHUNK_INSERT).returns(1)
|
147
|
+
@connection.expects(:execute).with(regexp_matches(/show warnings/),EXPECTED_RETRY_FLAGS_CHUNKER).returns([])
|
144
148
|
|
145
149
|
def @migration.conditions
|
146
150
|
"where foo.created_at > '2013-07-10' or foo.baz = 'quux'"
|
@@ -158,9 +162,9 @@ describe Lhm::Chunker do
|
|
158
162
|
2
|
159
163
|
end
|
160
164
|
|
161
|
-
@connection.expects(:select_value).with(regexp_matches(/where id >= 1 order by id limit 1 offset 1/),
|
162
|
-
@connection.expects(:update).with(regexp_matches(/inner join bar on foo.id = bar.foo_id and/),
|
163
|
-
@connection.expects(:execute).with(regexp_matches(/show warnings/),
|
165
|
+
@connection.expects(:select_value).with(regexp_matches(/where id >= 1 order by id limit 1 offset 1/),EXPECTED_RETRY_FLAGS_CHUNKER).returns(2)
|
166
|
+
@connection.expects(:update).with(regexp_matches(/inner join bar on foo.id = bar.foo_id and/),EXPECTED_RETRY_FLAGS_CHUNK_INSERT).returns(1)
|
167
|
+
@connection.expects(:execute).with(regexp_matches(/show warnings/),EXPECTED_RETRY_FLAGS_CHUNKER).returns([])
|
164
168
|
|
165
169
|
def @migration.conditions
|
166
170
|
'inner join bar on foo.id = bar.foo_id'
|
@@ -1,4 +1,5 @@
|
|
1
1
|
require 'lhm/connection'
|
2
|
+
require 'lhm/proxysql_helper'
|
2
3
|
|
3
4
|
describe Lhm::Connection do
|
4
5
|
|
@@ -12,10 +13,15 @@ describe Lhm::Connection do
|
|
12
13
|
it "Should find use calling file as prefix" do
|
13
14
|
ar_connection = mock()
|
14
15
|
ar_connection.stubs(:execute).raises(LOCK_WAIT).then.returns(true)
|
16
|
+
ar_connection.stubs(:active?).returns(true)
|
15
17
|
|
16
|
-
connection = Lhm::Connection.new(connection: ar_connection
|
18
|
+
connection = Lhm::Connection.new(connection: ar_connection, options: {
|
19
|
+
retriable: {
|
20
|
+
base_interval: 0
|
21
|
+
}
|
22
|
+
})
|
17
23
|
|
18
|
-
connection.execute("SHOW TABLES",
|
24
|
+
connection.execute("SHOW TABLES", should_retry: true)
|
19
25
|
|
20
26
|
log_messages = @logs.string.split("\n")
|
21
27
|
assert_equal(1, log_messages.length)
|
@@ -27,10 +33,16 @@ describe Lhm::Connection do
|
|
27
33
|
ar_connection.stubs(:execute).raises(LOCK_WAIT)
|
28
34
|
.then.raises(LOCK_WAIT)
|
29
35
|
.then.returns(true)
|
36
|
+
ar_connection.stubs(:active?).returns(true)
|
30
37
|
|
31
|
-
connection = Lhm::Connection.new(connection: ar_connection
|
38
|
+
connection = Lhm::Connection.new(connection: ar_connection, options: {
|
39
|
+
retriable: {
|
40
|
+
base_interval: 0,
|
41
|
+
tries: 3
|
42
|
+
}
|
43
|
+
})
|
32
44
|
|
33
|
-
connection.execute("SHOW TABLES",
|
45
|
+
connection.execute("SHOW TABLES", should_retry: true)
|
34
46
|
|
35
47
|
log_messages = @logs.string.split("\n")
|
36
48
|
assert_equal(2, log_messages.length)
|
@@ -41,10 +53,16 @@ describe Lhm::Connection do
|
|
41
53
|
ar_connection.stubs(:update).raises(LOCK_WAIT)
|
42
54
|
.then.raises(LOCK_WAIT)
|
43
55
|
.then.returns(1)
|
56
|
+
ar_connection.stubs(:active?).returns(true)
|
44
57
|
|
45
|
-
connection = Lhm::Connection.new(connection: ar_connection
|
58
|
+
connection = Lhm::Connection.new(connection: ar_connection, options: {
|
59
|
+
retriable: {
|
60
|
+
base_interval: 0,
|
61
|
+
tries: 3
|
62
|
+
}
|
63
|
+
})
|
46
64
|
|
47
|
-
val = connection.update("SHOW TABLES",
|
65
|
+
val = connection.update("SHOW TABLES", should_retry: true)
|
48
66
|
|
49
67
|
log_messages = @logs.string.split("\n")
|
50
68
|
assert_equal val, 1
|
@@ -56,13 +74,38 @@ describe Lhm::Connection do
|
|
56
74
|
ar_connection.stubs(:select_value).raises(LOCK_WAIT)
|
57
75
|
.then.raises(LOCK_WAIT)
|
58
76
|
.then.returns("dummy")
|
77
|
+
ar_connection.stubs(:active?).returns(true)
|
59
78
|
|
60
|
-
connection = Lhm::Connection.new(connection: ar_connection
|
79
|
+
connection = Lhm::Connection.new(connection: ar_connection, options: {
|
80
|
+
retriable: {
|
81
|
+
base_interval: 0,
|
82
|
+
tries: 3
|
83
|
+
}
|
84
|
+
})
|
61
85
|
|
62
|
-
val = connection.select_value("SHOW TABLES",
|
86
|
+
val = connection.select_value("SHOW TABLES", should_retry: true)
|
63
87
|
|
64
88
|
log_messages = @logs.string.split("\n")
|
65
89
|
assert_equal val, "dummy"
|
66
90
|
assert_equal(2, log_messages.length)
|
67
91
|
end
|
92
|
+
|
93
|
+
it "Queries should be tagged with ProxySQL tag if reconnect_with_consistent_host is enabled" do
|
94
|
+
ar_connection = mock()
|
95
|
+
ar_connection.expects(:public_send).with(:select_value, "SHOW TABLES #{Lhm::ProxySQLHelper::ANNOTATION}").returns("dummy")
|
96
|
+
ar_connection.stubs(:execute).times(4).returns([["dummy"]])
|
97
|
+
ar_connection.stubs(:active?).returns(true)
|
98
|
+
|
99
|
+
connection = Lhm::Connection.new(connection: ar_connection, options: {
|
100
|
+
reconnect_with_consistent_host: true,
|
101
|
+
retriable: {
|
102
|
+
base_interval: 0,
|
103
|
+
tries: 3
|
104
|
+
}
|
105
|
+
})
|
106
|
+
|
107
|
+
val = connection.select_value("SHOW TABLES", should_retry: true)
|
108
|
+
|
109
|
+
assert_equal val, "dummy"
|
110
|
+
end
|
68
111
|
end
|
data/spec/unit/entangler_spec.rb
CHANGED
@@ -63,42 +63,94 @@ describe Lhm::Entangler do
|
|
63
63
|
it 'should retry trigger creation when it hits a lock wait timeout' do
|
64
64
|
tries = 1
|
65
65
|
ar_connection = mock()
|
66
|
-
ar_connection.
|
67
|
-
|
68
|
-
|
69
|
-
|
70
|
-
|
66
|
+
ar_connection.stubs(:execute)
|
67
|
+
.returns([["dummy"]], [["dummy"]], [["dummy"]])
|
68
|
+
.then
|
69
|
+
.raises(Mysql2::Error, 'Lock wait timeout exceeded; try restarting transaction')
|
70
|
+
ar_connection.stubs(:active?).returns(true)
|
71
|
+
|
72
|
+
connection = Lhm::Connection.new(connection: ar_connection, options: {
|
73
|
+
reconnect_with_consistent_host: true,
|
74
|
+
retriable: {
|
75
|
+
base_interval: 0,
|
76
|
+
tries: tries
|
77
|
+
}
|
78
|
+
})
|
79
|
+
|
80
|
+
@entangler = Lhm::Entangler.new(@migration, connection)
|
71
81
|
|
72
82
|
assert_raises(Mysql2::Error) { @entangler.before }
|
73
83
|
end
|
74
84
|
|
75
85
|
it 'should not retry trigger creation with other mysql errors' do
|
76
86
|
ar_connection = mock()
|
77
|
-
ar_connection.
|
78
|
-
|
79
|
-
|
80
|
-
|
87
|
+
ar_connection.stubs(:execute)
|
88
|
+
.returns([["dummy"]], [["dummy"]], [["dummy"]])
|
89
|
+
.then
|
90
|
+
.raises(Mysql2::Error, 'The MySQL server is running with the --read-only option so it cannot execute this statement.')
|
91
|
+
ar_connection.stubs(:active?).returns(true)
|
92
|
+
connection = Lhm::Connection.new(connection: ar_connection, options: {
|
93
|
+
reconnect_with_consistent_host: true,
|
94
|
+
retriable: {
|
95
|
+
base_interval: 0
|
96
|
+
},
|
97
|
+
})
|
98
|
+
|
99
|
+
@entangler = Lhm::Entangler.new(@migration, connection)
|
81
100
|
assert_raises(Mysql2::Error) { @entangler.before }
|
82
101
|
end
|
83
102
|
|
84
103
|
it 'should succesfully finish after retrying' do
|
85
104
|
ar_connection = mock()
|
86
|
-
ar_connection.stubs(:execute)
|
87
|
-
|
88
|
-
|
89
|
-
|
90
|
-
|
105
|
+
ar_connection.stubs(:execute)
|
106
|
+
.returns([["dummy"]], [["dummy"]], [["dummy"]])
|
107
|
+
.then
|
108
|
+
.raises(Mysql2::Error, 'Lock wait timeout exceeded; try restarting transaction')
|
109
|
+
.then
|
110
|
+
.returns([["dummy"]])
|
111
|
+
ar_connection.stubs(:active?).returns(true)
|
112
|
+
|
113
|
+
connection = Lhm::Connection.new(connection: ar_connection, options: {
|
114
|
+
reconnect_with_consistent_host: true,
|
115
|
+
retriable: {
|
116
|
+
base_interval: 0
|
117
|
+
},
|
118
|
+
})
|
119
|
+
|
120
|
+
@entangler = Lhm::Entangler.new(@migration, connection)
|
91
121
|
|
92
122
|
assert @entangler.before
|
93
123
|
end
|
94
124
|
|
95
125
|
it 'should retry as many times as specified by configuration' do
|
96
126
|
ar_connection = mock()
|
97
|
-
ar_connection.
|
98
|
-
|
99
|
-
|
100
|
-
|
101
|
-
|
127
|
+
ar_connection.stubs(:execute)
|
128
|
+
.returns([["dummy"]], [["dummy"]], [["dummy"]]) # initial
|
129
|
+
.then
|
130
|
+
.raises(Mysql2::Error, 'Lock wait timeout exceeded; try restarting transaction')
|
131
|
+
.then
|
132
|
+
.returns([["dummy"]]) # reconnect 1
|
133
|
+
.then
|
134
|
+
.raises(Mysql2::Error, 'Lock wait timeout exceeded; try restarting transaction')
|
135
|
+
.then
|
136
|
+
.returns([["dummy"]]) # reconnect 2
|
137
|
+
.then
|
138
|
+
.raises(Mysql2::Error, 'Lock wait timeout exceeded; try restarting transaction')
|
139
|
+
.then
|
140
|
+
.returns([["dummy"]]) # reconnect 3
|
141
|
+
.then
|
142
|
+
.raises(Mysql2::Error, 'Lock wait timeout exceeded; try restarting transaction') # final error
|
143
|
+
ar_connection.stubs(:active?).returns(true)
|
144
|
+
|
145
|
+
connection = Lhm::Connection.new(connection: ar_connection, options: {
|
146
|
+
reconnect_with_consistent_host: true,
|
147
|
+
retriable: {
|
148
|
+
tries: 5,
|
149
|
+
base_interval: 0
|
150
|
+
},
|
151
|
+
})
|
152
|
+
|
153
|
+
@entangler = Lhm::Entangler.new(@migration, connection)
|
102
154
|
|
103
155
|
assert_raises(Mysql2::Error) { @entangler.before }
|
104
156
|
end
|
data/spec/unit/lhm_spec.rb
CHANGED
@@ -26,4 +26,21 @@ describe Lhm do
|
|
26
26
|
value(Lhm.logger.instance_eval { @logdev }.dev.path).must_equal 'omg.ponies'
|
27
27
|
end
|
28
28
|
end
|
29
|
+
|
30
|
+
describe 'api' do
|
31
|
+
|
32
|
+
before(:each) do
|
33
|
+
@connection = mock()
|
34
|
+
end
|
35
|
+
|
36
|
+
it 'should create a new connection when calling setup' do
|
37
|
+
Lhm.setup(@connection)
|
38
|
+
value(Lhm.connection).must_be_kind_of(Lhm::Connection)
|
39
|
+
end
|
40
|
+
|
41
|
+
it 'should create a new connection when none is created' do
|
42
|
+
ActiveRecord::Base.stubs(:connection).returns(@connection)
|
43
|
+
value(Lhm.connection).must_be_kind_of(Lhm::Connection)
|
44
|
+
end
|
45
|
+
end
|
29
46
|
end
|
@@ -39,7 +39,7 @@ describe Lhm::Throttler::Slave do
|
|
39
39
|
@logs = StringIO.new
|
40
40
|
Lhm.logger = Logger.new(@logs)
|
41
41
|
|
42
|
-
@dummy_mysql_client_config = lambda { {'username' => 'user', 'password' => 'pw', 'database' => 'db'} }
|
42
|
+
@dummy_mysql_client_config = lambda { { 'username' => 'user', 'password' => 'pw', 'database' => 'db' } }
|
43
43
|
end
|
44
44
|
|
45
45
|
describe "#client" do
|
@@ -64,7 +64,7 @@ describe Lhm::Throttler::Slave do
|
|
64
64
|
|
65
65
|
describe 'with proper config' do
|
66
66
|
it "creates a new Mysql2::Client" do
|
67
|
-
expected_config = {username: 'user', password: 'pw', database: 'db', host: 'slave'}
|
67
|
+
expected_config = { username: 'user', password: 'pw', database: 'db', host: 'slave' }
|
68
68
|
Mysql2::Client.stubs(:new).with(expected_config).returns(mock())
|
69
69
|
|
70
70
|
assert Lhm::Throttler::Slave.new('slave', @dummy_mysql_client_config).connection
|
@@ -73,8 +73,12 @@ describe Lhm::Throttler::Slave do
|
|
73
73
|
|
74
74
|
describe 'with active record config' do
|
75
75
|
it 'logs and creates client' do
|
76
|
-
active_record_config = {username: 'user', password: 'pw', database: 'db'}
|
77
|
-
ActiveRecord::
|
76
|
+
active_record_config = { username: 'user', password: 'pw', database: 'db' }
|
77
|
+
if ActiveRecord::VERSION::MAJOR > 6 || ActiveRecord::VERSION::MAJOR == 6 && ActiveRecord::VERSION::MINOR >= 1
|
78
|
+
ActiveRecord::Base.stubs(:connection_pool).returns(stub(db_config: stub(configuration_hash: active_record_config)))
|
79
|
+
else
|
80
|
+
ActiveRecord::Base.stubs(:connection_pool).returns(stub(spec: stub(config: active_record_config)))
|
81
|
+
end
|
78
82
|
|
79
83
|
Mysql2::Client.stubs(:new).returns(mock())
|
80
84
|
|
@@ -92,9 +96,9 @@ describe Lhm::Throttler::Slave do
|
|
92
96
|
class Connection
|
93
97
|
def self.query(query)
|
94
98
|
if query == Lhm::Throttler::Slave::SQL_SELECT_MAX_SLAVE_LAG
|
95
|
-
[{'Seconds_Behind_Master' => 20}]
|
99
|
+
[{ 'Seconds_Behind_Master' => 20 }]
|
96
100
|
elsif query == Lhm::Throttler::Slave::SQL_SELECT_SLAVE_HOSTS
|
97
|
-
[{'host' => '1.1.1.1:80'}]
|
101
|
+
[{ 'host' => '1.1.1.1:80' }]
|
98
102
|
end
|
99
103
|
end
|
100
104
|
end
|
@@ -104,7 +108,7 @@ describe Lhm::Throttler::Slave do
|
|
104
108
|
|
105
109
|
class StoppedConnection
|
106
110
|
def self.query(query)
|
107
|
-
[{'Seconds_Behind_Master' => nil}]
|
111
|
+
[{ 'Seconds_Behind_Master' => nil }]
|
108
112
|
end
|
109
113
|
end
|
110
114
|
|
@@ -138,7 +142,7 @@ describe Lhm::Throttler::Slave do
|
|
138
142
|
Lhm::Throttler::Slave.any_instance.stubs(:config).returns([])
|
139
143
|
|
140
144
|
slave = Lhm::Throttler::Slave.new('slave', @dummy_mysql_client_config)
|
141
|
-
|
145
|
+
Logger.any_instance.expects(:info).with("Unable to connect and/or query slave: Can't connect to MySQL server")
|
142
146
|
assert_equal(0, slave.lag)
|
143
147
|
end
|
144
148
|
end
|
@@ -286,7 +290,7 @@ describe Lhm::Throttler::SlaveLag do
|
|
286
290
|
describe 'with the :check_only option' do
|
287
291
|
describe 'with a callable argument' do
|
288
292
|
before do
|
289
|
-
check_only = lambda {{'host' => '1.1.1.3'}}
|
293
|
+
check_only = lambda { { 'host' => '1.1.1.3' } }
|
290
294
|
@throttler = Lhm::Throttler::SlaveLag.new :check_only => check_only
|
291
295
|
end
|
292
296
|
|
@@ -300,6 +304,7 @@ describe Lhm::Throttler::SlaveLag do
|
|
300
304
|
describe 'with a non-callable argument' do
|
301
305
|
before do
|
302
306
|
@throttler = Lhm::Throttler::SlaveLag.new :check_only => 'I cannot be called'
|
307
|
+
|
303
308
|
def @throttler.master_slave_hosts
|
304
309
|
['1.1.1.1', '1.1.1.4']
|
305
310
|
end
|