lhm-shopify 3.5.0 → 3.5.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.github/workflows/test.yml +17 -4
- data/.gitignore +0 -2
- data/Appraisals +24 -0
- data/Gemfile.lock +66 -0
- data/README.md +42 -0
- data/Rakefile +1 -0
- data/dev.yml +18 -3
- data/docker-compose.yml +15 -3
- data/gemfiles/activerecord_5.2.gemfile +9 -0
- data/gemfiles/activerecord_5.2.gemfile.lock +65 -0
- data/gemfiles/activerecord_6.0.gemfile +7 -0
- data/gemfiles/activerecord_6.0.gemfile.lock +67 -0
- data/gemfiles/activerecord_6.1.gemfile +7 -0
- data/gemfiles/activerecord_6.1.gemfile.lock +66 -0
- data/gemfiles/activerecord_7.0.0.alpha2.gemfile +7 -0
- data/gemfiles/activerecord_7.0.0.alpha2.gemfile.lock +64 -0
- data/lhm.gemspec +7 -3
- data/lib/lhm/atomic_switcher.rb +1 -1
- data/lib/lhm/chunk_insert.rb +2 -1
- data/lib/lhm/chunker.rb +3 -3
- data/lib/lhm/cleanup/current.rb +1 -1
- data/lib/lhm/connection.rb +50 -11
- data/lib/lhm/entangler.rb +2 -2
- data/lib/lhm/invoker.rb +2 -2
- data/lib/lhm/proxysql_helper.rb +10 -0
- data/lib/lhm/sql_retry.rb +126 -8
- data/lib/lhm/throttler/slave_lag.rb +19 -2
- data/lib/lhm/version.rb +1 -1
- data/lib/lhm.rb +22 -8
- data/scripts/mysql/writer/create_users.sql +3 -0
- data/spec/integration/atomic_switcher_spec.rb +27 -10
- data/spec/integration/chunk_insert_spec.rb +2 -1
- data/spec/integration/chunker_spec.rb +1 -1
- data/spec/integration/database.yml +10 -0
- data/spec/integration/entangler_spec.rb +3 -1
- data/spec/integration/integration_helper.rb +23 -5
- data/spec/integration/lhm_spec.rb +75 -0
- data/spec/integration/proxysql_spec.rb +34 -0
- data/spec/integration/sql_retry/db_connection_helper.rb +52 -0
- data/spec/integration/sql_retry/lock_wait_spec.rb +8 -6
- data/spec/integration/sql_retry/lock_wait_timeout_test_helper.rb +19 -9
- data/spec/integration/sql_retry/proxysql_helper.rb +22 -0
- data/spec/integration/sql_retry/retry_with_proxysql_spec.rb +108 -0
- data/spec/integration/toxiproxy_helper.rb +40 -0
- data/spec/test_helper.rb +21 -0
- data/spec/unit/chunk_insert_spec.rb +7 -2
- data/spec/unit/chunker_spec.rb +45 -42
- data/spec/unit/connection_spec.rb +22 -4
- data/spec/unit/entangler_spec.rb +41 -11
- data/spec/unit/throttler/slave_lag_spec.rb +13 -8
- metadata +76 -11
- data/gemfiles/ar-2.3_mysql.gemfile +0 -6
- data/gemfiles/ar-3.2_mysql.gemfile +0 -5
- data/gemfiles/ar-3.2_mysql2.gemfile +0 -5
- data/gemfiles/ar-4.0_mysql2.gemfile +0 -5
- data/gemfiles/ar-4.1_mysql2.gemfile +0 -5
- data/gemfiles/ar-4.2_mysql2.gemfile +0 -5
- data/gemfiles/ar-5.0_mysql2.gemfile +0 -5
data/spec/unit/chunker_spec.rb
CHANGED
@@ -12,11 +12,14 @@ require 'lhm/connection'
|
|
12
12
|
describe Lhm::Chunker do
|
13
13
|
include UnitHelper
|
14
14
|
|
15
|
+
EXPECTED_RETRY_FLAGS = {:should_retry => true, :retry_options => {}}
|
16
|
+
|
15
17
|
before(:each) do
|
16
18
|
@origin = Lhm::Table.new('foo')
|
17
19
|
@destination = Lhm::Table.new('bar')
|
18
20
|
@migration = Lhm::Migration.new(@origin, @destination)
|
19
21
|
@connection = mock()
|
22
|
+
@connection.stubs(:execute).returns([["dummy"]])
|
20
23
|
# This is a poor man's stub
|
21
24
|
@throttler = Object.new
|
22
25
|
def @throttler.run
|
@@ -38,11 +41,11 @@ describe Lhm::Chunker do
|
|
38
41
|
5
|
39
42
|
end
|
40
43
|
|
41
|
-
@connection.expects(:select_value).with(regexp_matches(/where id >= 1 order by id limit 1 offset 4/),
|
42
|
-
@connection.expects(:select_value).with(regexp_matches(/where id >= 8 order by id limit 1 offset 4/),
|
43
|
-
@connection.expects(:update).with(regexp_matches(/between 1 and 7/),
|
44
|
-
@connection.expects(:update).with(regexp_matches(/between 8 and 10/),
|
45
|
-
@connection.expects(:execute).twice.with(regexp_matches(/show warnings/),
|
44
|
+
@connection.expects(:select_value).with(regexp_matches(/where id >= 1 order by id limit 1 offset 4/),EXPECTED_RETRY_FLAGS).returns(7)
|
45
|
+
@connection.expects(:select_value).with(regexp_matches(/where id >= 8 order by id limit 1 offset 4/),EXPECTED_RETRY_FLAGS).returns(21)
|
46
|
+
@connection.expects(:update).with(regexp_matches(/between 1 and 7/),EXPECTED_RETRY_FLAGS).returns(2)
|
47
|
+
@connection.expects(:update).with(regexp_matches(/between 8 and 10/),EXPECTED_RETRY_FLAGS).returns(2)
|
48
|
+
@connection.expects(:execute).twice.with(regexp_matches(/show warnings/),EXPECTED_RETRY_FLAGS).returns([])
|
46
49
|
|
47
50
|
@chunker.run
|
48
51
|
end
|
@@ -53,17 +56,17 @@ describe Lhm::Chunker do
|
|
53
56
|
2
|
54
57
|
end
|
55
58
|
|
56
|
-
@connection.expects(:select_value).with(regexp_matches(/where id >= 1 order by id limit 1 offset 1/),
|
57
|
-
@connection.expects(:select_value).with(regexp_matches(/where id >= 3 order by id limit 1 offset 1/),
|
58
|
-
@connection.expects(:select_value).with(regexp_matches(/where id >= 5 order by id limit 1 offset 1/),
|
59
|
-
@connection.expects(:select_value).with(regexp_matches(/where id >= 7 order by id limit 1 offset 1/),
|
60
|
-
@connection.expects(:select_value).with(regexp_matches(/where id >= 9 order by id limit 1 offset 1/),
|
59
|
+
@connection.expects(:select_value).with(regexp_matches(/where id >= 1 order by id limit 1 offset 1/),EXPECTED_RETRY_FLAGS).returns(2)
|
60
|
+
@connection.expects(:select_value).with(regexp_matches(/where id >= 3 order by id limit 1 offset 1/),EXPECTED_RETRY_FLAGS).returns(4)
|
61
|
+
@connection.expects(:select_value).with(regexp_matches(/where id >= 5 order by id limit 1 offset 1/),EXPECTED_RETRY_FLAGS).returns(6)
|
62
|
+
@connection.expects(:select_value).with(regexp_matches(/where id >= 7 order by id limit 1 offset 1/),EXPECTED_RETRY_FLAGS).returns(8)
|
63
|
+
@connection.expects(:select_value).with(regexp_matches(/where id >= 9 order by id limit 1 offset 1/),EXPECTED_RETRY_FLAGS).returns(10)
|
61
64
|
|
62
|
-
@connection.expects(:update).with(regexp_matches(/between 1 and 2/),
|
63
|
-
@connection.expects(:update).with(regexp_matches(/between 3 and 4/),
|
64
|
-
@connection.expects(:update).with(regexp_matches(/between 5 and 6/),
|
65
|
-
@connection.expects(:update).with(regexp_matches(/between 7 and 8/),
|
66
|
-
@connection.expects(:update).with(regexp_matches(/between 9 and 10/),
|
65
|
+
@connection.expects(:update).with(regexp_matches(/between 1 and 2/),EXPECTED_RETRY_FLAGS).returns(2)
|
66
|
+
@connection.expects(:update).with(regexp_matches(/between 3 and 4/),EXPECTED_RETRY_FLAGS).returns(2)
|
67
|
+
@connection.expects(:update).with(regexp_matches(/between 5 and 6/),EXPECTED_RETRY_FLAGS).returns(2)
|
68
|
+
@connection.expects(:update).with(regexp_matches(/between 7 and 8/),EXPECTED_RETRY_FLAGS).returns(2)
|
69
|
+
@connection.expects(:update).with(regexp_matches(/between 9 and 10/),EXPECTED_RETRY_FLAGS).returns(2)
|
67
70
|
|
68
71
|
@chunker.run
|
69
72
|
end
|
@@ -80,17 +83,17 @@ describe Lhm::Chunker do
|
|
80
83
|
end
|
81
84
|
end
|
82
85
|
|
83
|
-
@connection.expects(:select_value).with(regexp_matches(/where id >= 1 order by id limit 1 offset 1/),
|
84
|
-
@connection.expects(:select_value).with(regexp_matches(/where id >= 3 order by id limit 1 offset 2/),
|
85
|
-
@connection.expects(:select_value).with(regexp_matches(/where id >= 6 order by id limit 1 offset 2/),
|
86
|
-
@connection.expects(:select_value).with(regexp_matches(/where id >= 9 order by id limit 1 offset 2/),
|
86
|
+
@connection.expects(:select_value).with(regexp_matches(/where id >= 1 order by id limit 1 offset 1/),EXPECTED_RETRY_FLAGS).returns(2)
|
87
|
+
@connection.expects(:select_value).with(regexp_matches(/where id >= 3 order by id limit 1 offset 2/),EXPECTED_RETRY_FLAGS).returns(5)
|
88
|
+
@connection.expects(:select_value).with(regexp_matches(/where id >= 6 order by id limit 1 offset 2/),EXPECTED_RETRY_FLAGS).returns(8)
|
89
|
+
@connection.expects(:select_value).with(regexp_matches(/where id >= 9 order by id limit 1 offset 2/),EXPECTED_RETRY_FLAGS).returns(nil)
|
87
90
|
|
88
|
-
@connection.expects(:update).with(regexp_matches(/between 1 and 2/),
|
89
|
-
@connection.expects(:update).with(regexp_matches(/between 3 and 5/),
|
90
|
-
@connection.expects(:update).with(regexp_matches(/between 6 and 8/),
|
91
|
-
@connection.expects(:update).with(regexp_matches(/between 9 and 10/),
|
91
|
+
@connection.expects(:update).with(regexp_matches(/between 1 and 2/),EXPECTED_RETRY_FLAGS).returns(2)
|
92
|
+
@connection.expects(:update).with(regexp_matches(/between 3 and 5/),EXPECTED_RETRY_FLAGS).returns(2)
|
93
|
+
@connection.expects(:update).with(regexp_matches(/between 6 and 8/),EXPECTED_RETRY_FLAGS).returns(2)
|
94
|
+
@connection.expects(:update).with(regexp_matches(/between 9 and 10/),EXPECTED_RETRY_FLAGS).returns(2)
|
92
95
|
|
93
|
-
@connection.expects(:execute).twice.with(regexp_matches(/show warnings/),
|
96
|
+
@connection.expects(:execute).twice.with(regexp_matches(/show warnings/),EXPECTED_RETRY_FLAGS).returns([])
|
94
97
|
|
95
98
|
@chunker.run
|
96
99
|
end
|
@@ -100,8 +103,8 @@ describe Lhm::Chunker do
|
|
100
103
|
:start => 1,
|
101
104
|
:limit => 1)
|
102
105
|
|
103
|
-
@connection.expects(:select_value).with(regexp_matches(/where id >= 1 order by id limit 1 offset 0/),
|
104
|
-
@connection.expects(:update).with(regexp_matches(/between 1 and 1/),
|
106
|
+
@connection.expects(:select_value).with(regexp_matches(/where id >= 1 order by id limit 1 offset 0/),EXPECTED_RETRY_FLAGS).returns(nil)
|
107
|
+
@connection.expects(:update).with(regexp_matches(/between 1 and 1/),EXPECTED_RETRY_FLAGS).returns(1)
|
105
108
|
|
106
109
|
@chunker.run
|
107
110
|
end
|
@@ -114,17 +117,17 @@ describe Lhm::Chunker do
|
|
114
117
|
2
|
115
118
|
end
|
116
119
|
|
117
|
-
@connection.expects(:select_value).with(regexp_matches(/where id >= 2 order by id limit 1 offset 1/),
|
118
|
-
@connection.expects(:select_value).with(regexp_matches(/where id >= 4 order by id limit 1 offset 1/),
|
119
|
-
@connection.expects(:select_value).with(regexp_matches(/where id >= 6 order by id limit 1 offset 1/),
|
120
|
-
@connection.expects(:select_value).with(regexp_matches(/where id >= 8 order by id limit 1 offset 1/),
|
121
|
-
@connection.expects(:select_value).with(regexp_matches(/where id >= 10 order by id limit 1 offset 1/),
|
120
|
+
@connection.expects(:select_value).with(regexp_matches(/where id >= 2 order by id limit 1 offset 1/),EXPECTED_RETRY_FLAGS).returns(3)
|
121
|
+
@connection.expects(:select_value).with(regexp_matches(/where id >= 4 order by id limit 1 offset 1/),EXPECTED_RETRY_FLAGS).returns(5)
|
122
|
+
@connection.expects(:select_value).with(regexp_matches(/where id >= 6 order by id limit 1 offset 1/),EXPECTED_RETRY_FLAGS).returns(7)
|
123
|
+
@connection.expects(:select_value).with(regexp_matches(/where id >= 8 order by id limit 1 offset 1/),EXPECTED_RETRY_FLAGS).returns(9)
|
124
|
+
@connection.expects(:select_value).with(regexp_matches(/where id >= 10 order by id limit 1 offset 1/),EXPECTED_RETRY_FLAGS).returns(nil)
|
122
125
|
|
123
|
-
@connection.expects(:update).with(regexp_matches(/between 2 and 3/),
|
124
|
-
@connection.expects(:update).with(regexp_matches(/between 4 and 5/),
|
125
|
-
@connection.expects(:update).with(regexp_matches(/between 6 and 7/),
|
126
|
-
@connection.expects(:update).with(regexp_matches(/between 8 and 9/),
|
127
|
-
@connection.expects(:update).with(regexp_matches(/between 10 and 10/),
|
126
|
+
@connection.expects(:update).with(regexp_matches(/between 2 and 3/),EXPECTED_RETRY_FLAGS).returns(2)
|
127
|
+
@connection.expects(:update).with(regexp_matches(/between 4 and 5/),EXPECTED_RETRY_FLAGS).returns(2)
|
128
|
+
@connection.expects(:update).with(regexp_matches(/between 6 and 7/),EXPECTED_RETRY_FLAGS).returns(2)
|
129
|
+
@connection.expects(:update).with(regexp_matches(/between 8 and 9/),EXPECTED_RETRY_FLAGS).returns(2)
|
130
|
+
@connection.expects(:update).with(regexp_matches(/between 10 and 10/),EXPECTED_RETRY_FLAGS).returns(1)
|
128
131
|
|
129
132
|
@chunker.run
|
130
133
|
end
|
@@ -138,9 +141,9 @@ describe Lhm::Chunker do
|
|
138
141
|
2
|
139
142
|
end
|
140
143
|
|
141
|
-
@connection.expects(:select_value).with(regexp_matches(/where id >= 1 order by id limit 1 offset 1/),
|
142
|
-
@connection.expects(:update).with(regexp_matches(/where \(foo.created_at > '2013-07-10' or foo.baz = 'quux'\) and `foo`/),
|
143
|
-
@connection.expects(:execute).with(regexp_matches(/show warnings/),
|
144
|
+
@connection.expects(:select_value).with(regexp_matches(/where id >= 1 order by id limit 1 offset 1/),EXPECTED_RETRY_FLAGS).returns(2)
|
145
|
+
@connection.expects(:update).with(regexp_matches(/where \(foo.created_at > '2013-07-10' or foo.baz = 'quux'\) and `foo`/),EXPECTED_RETRY_FLAGS).returns(1)
|
146
|
+
@connection.expects(:execute).with(regexp_matches(/show warnings/),EXPECTED_RETRY_FLAGS).returns([])
|
144
147
|
|
145
148
|
def @migration.conditions
|
146
149
|
"where foo.created_at > '2013-07-10' or foo.baz = 'quux'"
|
@@ -158,9 +161,9 @@ describe Lhm::Chunker do
|
|
158
161
|
2
|
159
162
|
end
|
160
163
|
|
161
|
-
@connection.expects(:select_value).with(regexp_matches(/where id >= 1 order by id limit 1 offset 1/),
|
162
|
-
@connection.expects(:update).with(regexp_matches(/inner join bar on foo.id = bar.foo_id and/),
|
163
|
-
@connection.expects(:execute).with(regexp_matches(/show warnings/),
|
164
|
+
@connection.expects(:select_value).with(regexp_matches(/where id >= 1 order by id limit 1 offset 1/),EXPECTED_RETRY_FLAGS).returns(2)
|
165
|
+
@connection.expects(:update).with(regexp_matches(/inner join bar on foo.id = bar.foo_id and/),EXPECTED_RETRY_FLAGS).returns(1)
|
166
|
+
@connection.expects(:execute).with(regexp_matches(/show warnings/),EXPECTED_RETRY_FLAGS).returns([])
|
164
167
|
|
165
168
|
def @migration.conditions
|
166
169
|
'inner join bar on foo.id = bar.foo_id'
|
@@ -1,4 +1,5 @@
|
|
1
1
|
require 'lhm/connection'
|
2
|
+
require 'lhm/proxysql_helper'
|
2
3
|
|
3
4
|
describe Lhm::Connection do
|
4
5
|
|
@@ -12,10 +13,11 @@ describe Lhm::Connection do
|
|
12
13
|
it "Should find use calling file as prefix" do
|
13
14
|
ar_connection = mock()
|
14
15
|
ar_connection.stubs(:execute).raises(LOCK_WAIT).then.returns(true)
|
16
|
+
ar_connection.stubs(:active?).returns(true)
|
15
17
|
|
16
18
|
connection = Lhm::Connection.new(connection: ar_connection)
|
17
19
|
|
18
|
-
connection.execute("SHOW TABLES", { base_interval: 0 })
|
20
|
+
connection.execute("SHOW TABLES", should_retry: true, retry_options: { base_interval: 0 })
|
19
21
|
|
20
22
|
log_messages = @logs.string.split("\n")
|
21
23
|
assert_equal(1, log_messages.length)
|
@@ -27,10 +29,11 @@ describe Lhm::Connection do
|
|
27
29
|
ar_connection.stubs(:execute).raises(LOCK_WAIT)
|
28
30
|
.then.raises(LOCK_WAIT)
|
29
31
|
.then.returns(true)
|
32
|
+
ar_connection.stubs(:active?).returns(true)
|
30
33
|
|
31
34
|
connection = Lhm::Connection.new(connection: ar_connection)
|
32
35
|
|
33
|
-
connection.execute("SHOW TABLES", { base_interval: 0, tries: 3 })
|
36
|
+
connection.execute("SHOW TABLES", should_retry: true, retry_options: { base_interval: 0, tries: 3 })
|
34
37
|
|
35
38
|
log_messages = @logs.string.split("\n")
|
36
39
|
assert_equal(2, log_messages.length)
|
@@ -41,10 +44,11 @@ describe Lhm::Connection do
|
|
41
44
|
ar_connection.stubs(:update).raises(LOCK_WAIT)
|
42
45
|
.then.raises(LOCK_WAIT)
|
43
46
|
.then.returns(1)
|
47
|
+
ar_connection.stubs(:active?).returns(true)
|
44
48
|
|
45
49
|
connection = Lhm::Connection.new(connection: ar_connection)
|
46
50
|
|
47
|
-
val = connection.update("SHOW TABLES", { base_interval: 0, tries: 3 })
|
51
|
+
val = connection.update("SHOW TABLES", should_retry: true, retry_options:{ base_interval: 0, tries: 3 })
|
48
52
|
|
49
53
|
log_messages = @logs.string.split("\n")
|
50
54
|
assert_equal val, 1
|
@@ -56,13 +60,27 @@ describe Lhm::Connection do
|
|
56
60
|
ar_connection.stubs(:select_value).raises(LOCK_WAIT)
|
57
61
|
.then.raises(LOCK_WAIT)
|
58
62
|
.then.returns("dummy")
|
63
|
+
ar_connection.stubs(:active?).returns(true)
|
59
64
|
|
60
65
|
connection = Lhm::Connection.new(connection: ar_connection)
|
61
66
|
|
62
|
-
val = connection.select_value("SHOW TABLES", { base_interval: 0, tries: 3 })
|
67
|
+
val = connection.select_value("SHOW TABLES", should_retry: true, retry_options: { base_interval: 0, tries: 3 })
|
63
68
|
|
64
69
|
log_messages = @logs.string.split("\n")
|
65
70
|
assert_equal val, "dummy"
|
66
71
|
assert_equal(2, log_messages.length)
|
67
72
|
end
|
73
|
+
|
74
|
+
it "Queries should be tagged with ProxySQL tag if requested" do
|
75
|
+
ar_connection = mock()
|
76
|
+
ar_connection.expects(:public_send).with(:select_value, "#{Lhm::ProxySQLHelper::ANNOTATION}SHOW TABLES").returns("dummy")
|
77
|
+
ar_connection.stubs(:execute).times(4).returns([["dummy"]])
|
78
|
+
ar_connection.stubs(:active?).returns(true)
|
79
|
+
|
80
|
+
connection = Lhm::Connection.new(connection: ar_connection, options: { reconnect_with_consistent_host: true })
|
81
|
+
|
82
|
+
val = connection.select_value("SHOW TABLES", should_retry: true, retry_options: { base_interval: 0, tries: 3 })
|
83
|
+
|
84
|
+
assert_equal val, "dummy"
|
85
|
+
end
|
68
86
|
end
|
data/spec/unit/entangler_spec.rb
CHANGED
@@ -63,9 +63,13 @@ describe Lhm::Entangler do
|
|
63
63
|
it 'should retry trigger creation when it hits a lock wait timeout' do
|
64
64
|
tries = 1
|
65
65
|
ar_connection = mock()
|
66
|
-
ar_connection.
|
66
|
+
ar_connection.stubs(:execute)
|
67
|
+
.returns([["dummy"]], [["dummy"]], [["dummy"]])
|
68
|
+
.then
|
69
|
+
.raises(Mysql2::Error, 'Lock wait timeout exceeded; try restarting transaction')
|
70
|
+
ar_connection.stubs(:active?).returns(true)
|
67
71
|
|
68
|
-
connection = Lhm::Connection.new(connection: ar_connection)
|
72
|
+
connection = Lhm::Connection.new(connection: ar_connection, options: {reconnect_with_consistent_host: true})
|
69
73
|
|
70
74
|
@entangler = Lhm::Entangler.new(@migration, connection, retriable: {base_interval: 0, tries: tries})
|
71
75
|
|
@@ -74,18 +78,28 @@ describe Lhm::Entangler do
|
|
74
78
|
|
75
79
|
it 'should not retry trigger creation with other mysql errors' do
|
76
80
|
ar_connection = mock()
|
77
|
-
ar_connection.
|
78
|
-
|
79
|
-
|
80
|
-
|
81
|
+
ar_connection.stubs(:execute)
|
82
|
+
.returns([["dummy"]], [["dummy"]], [["dummy"]])
|
83
|
+
.then
|
84
|
+
.raises(Mysql2::Error, 'The MySQL server is running with the --read-only option so it cannot execute this statement.')
|
85
|
+
ar_connection.stubs(:active?).returns(true)
|
86
|
+
connection = Lhm::Connection.new(connection: ar_connection, options: {reconnect_with_consistent_host: true})
|
87
|
+
|
88
|
+
@entangler = Lhm::Entangler.new(@migration, connection, retriable: { base_interval: 0 })
|
81
89
|
assert_raises(Mysql2::Error) { @entangler.before }
|
82
90
|
end
|
83
91
|
|
84
92
|
it 'should succesfully finish after retrying' do
|
85
93
|
ar_connection = mock()
|
86
|
-
ar_connection.stubs(:execute)
|
94
|
+
ar_connection.stubs(:execute)
|
95
|
+
.returns([["dummy"]], [["dummy"]], [["dummy"]])
|
96
|
+
.then
|
97
|
+
.raises(Mysql2::Error, 'Lock wait timeout exceeded; try restarting transaction')
|
98
|
+
.then
|
99
|
+
.returns([["dummy"]])
|
100
|
+
ar_connection.stubs(:active?).returns(true)
|
87
101
|
|
88
|
-
connection = Lhm::Connection.new(connection: ar_connection)
|
102
|
+
connection = Lhm::Connection.new(connection: ar_connection, options: {reconnect_with_consistent_host: true})
|
89
103
|
|
90
104
|
@entangler = Lhm::Entangler.new(@migration, connection, retriable: {base_interval: 0})
|
91
105
|
|
@@ -94,9 +108,25 @@ describe Lhm::Entangler do
|
|
94
108
|
|
95
109
|
it 'should retry as many times as specified by configuration' do
|
96
110
|
ar_connection = mock()
|
97
|
-
ar_connection.
|
98
|
-
|
99
|
-
|
111
|
+
ar_connection.stubs(:execute)
|
112
|
+
.returns([["dummy"]], [["dummy"]], [["dummy"]]) # initial
|
113
|
+
.then
|
114
|
+
.raises(Mysql2::Error, 'Lock wait timeout exceeded; try restarting transaction')
|
115
|
+
.then
|
116
|
+
.returns([["dummy"]]) # reconnect 1
|
117
|
+
.then
|
118
|
+
.raises(Mysql2::Error, 'Lock wait timeout exceeded; try restarting transaction')
|
119
|
+
.then
|
120
|
+
.returns([["dummy"]]) # reconnect 2
|
121
|
+
.then
|
122
|
+
.raises(Mysql2::Error, 'Lock wait timeout exceeded; try restarting transaction')
|
123
|
+
.then
|
124
|
+
.returns([["dummy"]]) # reconnect 3
|
125
|
+
.then
|
126
|
+
.raises(Mysql2::Error, 'Lock wait timeout exceeded; try restarting transaction') # final error
|
127
|
+
ar_connection.stubs(:active?).returns(true)
|
128
|
+
|
129
|
+
connection = Lhm::Connection.new(connection: ar_connection, options: {reconnect_with_consistent_host: true})
|
100
130
|
|
101
131
|
@entangler = Lhm::Entangler.new(@migration, connection, retriable: {tries: 5, base_interval: 0})
|
102
132
|
|
@@ -39,7 +39,7 @@ describe Lhm::Throttler::Slave do
|
|
39
39
|
@logs = StringIO.new
|
40
40
|
Lhm.logger = Logger.new(@logs)
|
41
41
|
|
42
|
-
@dummy_mysql_client_config = lambda { {'username' => 'user', 'password' => 'pw', 'database' => 'db'} }
|
42
|
+
@dummy_mysql_client_config = lambda { { 'username' => 'user', 'password' => 'pw', 'database' => 'db' } }
|
43
43
|
end
|
44
44
|
|
45
45
|
describe "#client" do
|
@@ -64,7 +64,7 @@ describe Lhm::Throttler::Slave do
|
|
64
64
|
|
65
65
|
describe 'with proper config' do
|
66
66
|
it "creates a new Mysql2::Client" do
|
67
|
-
expected_config = {username: 'user', password: 'pw', database: 'db', host: 'slave'}
|
67
|
+
expected_config = { username: 'user', password: 'pw', database: 'db', host: 'slave' }
|
68
68
|
Mysql2::Client.stubs(:new).with(expected_config).returns(mock())
|
69
69
|
|
70
70
|
assert Lhm::Throttler::Slave.new('slave', @dummy_mysql_client_config).connection
|
@@ -73,8 +73,12 @@ describe Lhm::Throttler::Slave do
|
|
73
73
|
|
74
74
|
describe 'with active record config' do
|
75
75
|
it 'logs and creates client' do
|
76
|
-
active_record_config = {username: 'user', password: 'pw', database: 'db'}
|
77
|
-
ActiveRecord::
|
76
|
+
active_record_config = { username: 'user', password: 'pw', database: 'db' }
|
77
|
+
if ActiveRecord::VERSION::MAJOR > 6 || ActiveRecord::VERSION::MAJOR == 6 && ActiveRecord::VERSION::MINOR >= 1
|
78
|
+
ActiveRecord::Base.stubs(:connection_pool).returns(stub(db_config: stub(configuration_hash: active_record_config)))
|
79
|
+
else
|
80
|
+
ActiveRecord::Base.stubs(:connection_pool).returns(stub(spec: stub(config: active_record_config)))
|
81
|
+
end
|
78
82
|
|
79
83
|
Mysql2::Client.stubs(:new).returns(mock())
|
80
84
|
|
@@ -92,9 +96,9 @@ describe Lhm::Throttler::Slave do
|
|
92
96
|
class Connection
|
93
97
|
def self.query(query)
|
94
98
|
if query == Lhm::Throttler::Slave::SQL_SELECT_MAX_SLAVE_LAG
|
95
|
-
[{'Seconds_Behind_Master' => 20}]
|
99
|
+
[{ 'Seconds_Behind_Master' => 20 }]
|
96
100
|
elsif query == Lhm::Throttler::Slave::SQL_SELECT_SLAVE_HOSTS
|
97
|
-
[{'host' => '1.1.1.1:80'}]
|
101
|
+
[{ 'host' => '1.1.1.1:80' }]
|
98
102
|
end
|
99
103
|
end
|
100
104
|
end
|
@@ -104,7 +108,7 @@ describe Lhm::Throttler::Slave do
|
|
104
108
|
|
105
109
|
class StoppedConnection
|
106
110
|
def self.query(query)
|
107
|
-
[{'Seconds_Behind_Master' => nil}]
|
111
|
+
[{ 'Seconds_Behind_Master' => nil }]
|
108
112
|
end
|
109
113
|
end
|
110
114
|
|
@@ -286,7 +290,7 @@ describe Lhm::Throttler::SlaveLag do
|
|
286
290
|
describe 'with the :check_only option' do
|
287
291
|
describe 'with a callable argument' do
|
288
292
|
before do
|
289
|
-
check_only = lambda {{'host' => '1.1.1.3'}}
|
293
|
+
check_only = lambda { { 'host' => '1.1.1.3' } }
|
290
294
|
@throttler = Lhm::Throttler::SlaveLag.new :check_only => check_only
|
291
295
|
end
|
292
296
|
|
@@ -300,6 +304,7 @@ describe Lhm::Throttler::SlaveLag do
|
|
300
304
|
describe 'with a non-callable argument' do
|
301
305
|
before do
|
302
306
|
@throttler = Lhm::Throttler::SlaveLag.new :check_only => 'I cannot be called'
|
307
|
+
|
303
308
|
def @throttler.master_slave_hosts
|
304
309
|
['1.1.1.1', '1.1.1.4']
|
305
310
|
end
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: lhm-shopify
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 3.5.
|
4
|
+
version: 3.5.1
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- SoundCloud
|
@@ -12,7 +12,7 @@ authors:
|
|
12
12
|
autorequire:
|
13
13
|
bindir: bin
|
14
14
|
cert_chain: []
|
15
|
-
date: 2021-12-
|
15
|
+
date: 2021-12-06 00:00:00.000000000 Z
|
16
16
|
dependencies:
|
17
17
|
- !ruby/object:Gem::Dependency
|
18
18
|
name: retriable
|
@@ -28,6 +28,20 @@ dependencies:
|
|
28
28
|
- - ">="
|
29
29
|
- !ruby/object:Gem::Version
|
30
30
|
version: 3.0.0
|
31
|
+
- !ruby/object:Gem::Dependency
|
32
|
+
name: activerecord
|
33
|
+
requirement: !ruby/object:Gem::Requirement
|
34
|
+
requirements:
|
35
|
+
- - ">="
|
36
|
+
- !ruby/object:Gem::Version
|
37
|
+
version: '0'
|
38
|
+
type: :development
|
39
|
+
prerelease: false
|
40
|
+
version_requirements: !ruby/object:Gem::Requirement
|
41
|
+
requirements:
|
42
|
+
- - ">="
|
43
|
+
- !ruby/object:Gem::Version
|
44
|
+
version: '0'
|
31
45
|
- !ruby/object:Gem::Dependency
|
32
46
|
name: minitest
|
33
47
|
requirement: !ruby/object:Gem::Requirement
|
@@ -57,7 +71,7 @@ dependencies:
|
|
57
71
|
- !ruby/object:Gem::Version
|
58
72
|
version: '0'
|
59
73
|
- !ruby/object:Gem::Dependency
|
60
|
-
name:
|
74
|
+
name: after_do
|
61
75
|
requirement: !ruby/object:Gem::Requirement
|
62
76
|
requirements:
|
63
77
|
- - ">="
|
@@ -71,7 +85,7 @@ dependencies:
|
|
71
85
|
- !ruby/object:Gem::Version
|
72
86
|
version: '0'
|
73
87
|
- !ruby/object:Gem::Dependency
|
74
|
-
name:
|
88
|
+
name: rake
|
75
89
|
requirement: !ruby/object:Gem::Requirement
|
76
90
|
requirements:
|
77
91
|
- - ">="
|
@@ -112,6 +126,48 @@ dependencies:
|
|
112
126
|
- - ">="
|
113
127
|
- !ruby/object:Gem::Version
|
114
128
|
version: '0'
|
129
|
+
- !ruby/object:Gem::Dependency
|
130
|
+
name: toxiproxy
|
131
|
+
requirement: !ruby/object:Gem::Requirement
|
132
|
+
requirements:
|
133
|
+
- - ">="
|
134
|
+
- !ruby/object:Gem::Version
|
135
|
+
version: '0'
|
136
|
+
type: :development
|
137
|
+
prerelease: false
|
138
|
+
version_requirements: !ruby/object:Gem::Requirement
|
139
|
+
requirements:
|
140
|
+
- - ">="
|
141
|
+
- !ruby/object:Gem::Version
|
142
|
+
version: '0'
|
143
|
+
- !ruby/object:Gem::Dependency
|
144
|
+
name: appraisal
|
145
|
+
requirement: !ruby/object:Gem::Requirement
|
146
|
+
requirements:
|
147
|
+
- - ">="
|
148
|
+
- !ruby/object:Gem::Version
|
149
|
+
version: '0'
|
150
|
+
type: :development
|
151
|
+
prerelease: false
|
152
|
+
version_requirements: !ruby/object:Gem::Requirement
|
153
|
+
requirements:
|
154
|
+
- - ">="
|
155
|
+
- !ruby/object:Gem::Version
|
156
|
+
version: '0'
|
157
|
+
- !ruby/object:Gem::Dependency
|
158
|
+
name: byebug
|
159
|
+
requirement: !ruby/object:Gem::Requirement
|
160
|
+
requirements:
|
161
|
+
- - ">="
|
162
|
+
- !ruby/object:Gem::Version
|
163
|
+
version: '0'
|
164
|
+
type: :development
|
165
|
+
prerelease: false
|
166
|
+
version_requirements: !ruby/object:Gem::Requirement
|
167
|
+
requirements:
|
168
|
+
- - ">="
|
169
|
+
- !ruby/object:Gem::Version
|
170
|
+
version: '0'
|
115
171
|
description: Migrate large tables without downtime by copying to a temporary table
|
116
172
|
in chunks. The old table is not dropped. Instead, it is moved to timestamp_table_name
|
117
173
|
for verification.
|
@@ -124,20 +180,23 @@ files:
|
|
124
180
|
- ".gitignore"
|
125
181
|
- ".rubocop.yml"
|
126
182
|
- ".travis.yml"
|
183
|
+
- Appraisals
|
127
184
|
- CHANGELOG.md
|
128
185
|
- Gemfile
|
186
|
+
- Gemfile.lock
|
129
187
|
- LICENSE
|
130
188
|
- README.md
|
131
189
|
- Rakefile
|
132
190
|
- dev.yml
|
133
191
|
- docker-compose.yml
|
134
|
-
- gemfiles/
|
135
|
-
- gemfiles/
|
136
|
-
- gemfiles/
|
137
|
-
- gemfiles/
|
138
|
-
- gemfiles/
|
139
|
-
- gemfiles/
|
140
|
-
- gemfiles/
|
192
|
+
- gemfiles/activerecord_5.2.gemfile
|
193
|
+
- gemfiles/activerecord_5.2.gemfile.lock
|
194
|
+
- gemfiles/activerecord_6.0.gemfile
|
195
|
+
- gemfiles/activerecord_6.0.gemfile.lock
|
196
|
+
- gemfiles/activerecord_6.1.gemfile
|
197
|
+
- gemfiles/activerecord_6.1.gemfile.lock
|
198
|
+
- gemfiles/activerecord_7.0.0.alpha2.gemfile
|
199
|
+
- gemfiles/activerecord_7.0.0.alpha2.gemfile.lock
|
141
200
|
- lhm.gemspec
|
142
201
|
- lib/lhm-shopify.rb
|
143
202
|
- lib/lhm.rb
|
@@ -155,6 +214,7 @@ files:
|
|
155
214
|
- lib/lhm/migration.rb
|
156
215
|
- lib/lhm/migrator.rb
|
157
216
|
- lib/lhm/printer.rb
|
217
|
+
- lib/lhm/proxysql_helper.rb
|
158
218
|
- lib/lhm/railtie.rb
|
159
219
|
- lib/lhm/sql_helper.rb
|
160
220
|
- lib/lhm/sql_retry.rb
|
@@ -199,9 +259,14 @@ files:
|
|
199
259
|
- spec/integration/lhm_spec.rb
|
200
260
|
- spec/integration/lock_wait_timeout_spec.rb
|
201
261
|
- spec/integration/locked_switcher_spec.rb
|
262
|
+
- spec/integration/proxysql_spec.rb
|
263
|
+
- spec/integration/sql_retry/db_connection_helper.rb
|
202
264
|
- spec/integration/sql_retry/lock_wait_spec.rb
|
203
265
|
- spec/integration/sql_retry/lock_wait_timeout_test_helper.rb
|
266
|
+
- spec/integration/sql_retry/proxysql_helper.rb
|
267
|
+
- spec/integration/sql_retry/retry_with_proxysql_spec.rb
|
204
268
|
- spec/integration/table_spec.rb
|
269
|
+
- spec/integration/toxiproxy_helper.rb
|
205
270
|
- spec/test_helper.rb
|
206
271
|
- spec/unit/atomic_switcher_spec.rb
|
207
272
|
- spec/unit/chunk_finder_spec.rb
|