lhm-shopify 3.5.0 → 3.5.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.github/workflows/test.yml +17 -4
- data/.gitignore +0 -2
- data/Appraisals +24 -0
- data/Gemfile.lock +66 -0
- data/README.md +42 -0
- data/Rakefile +1 -0
- data/dev.yml +18 -3
- data/docker-compose.yml +15 -3
- data/gemfiles/activerecord_5.2.gemfile +9 -0
- data/gemfiles/activerecord_5.2.gemfile.lock +65 -0
- data/gemfiles/activerecord_6.0.gemfile +7 -0
- data/gemfiles/activerecord_6.0.gemfile.lock +67 -0
- data/gemfiles/activerecord_6.1.gemfile +7 -0
- data/gemfiles/activerecord_6.1.gemfile.lock +66 -0
- data/gemfiles/activerecord_7.0.0.alpha2.gemfile +7 -0
- data/gemfiles/activerecord_7.0.0.alpha2.gemfile.lock +64 -0
- data/lhm.gemspec +7 -3
- data/lib/lhm/atomic_switcher.rb +1 -1
- data/lib/lhm/chunk_insert.rb +2 -1
- data/lib/lhm/chunker.rb +3 -3
- data/lib/lhm/cleanup/current.rb +1 -1
- data/lib/lhm/connection.rb +50 -11
- data/lib/lhm/entangler.rb +2 -2
- data/lib/lhm/invoker.rb +2 -2
- data/lib/lhm/proxysql_helper.rb +10 -0
- data/lib/lhm/sql_retry.rb +126 -8
- data/lib/lhm/throttler/slave_lag.rb +19 -2
- data/lib/lhm/version.rb +1 -1
- data/lib/lhm.rb +22 -8
- data/scripts/mysql/writer/create_users.sql +3 -0
- data/spec/integration/atomic_switcher_spec.rb +27 -10
- data/spec/integration/chunk_insert_spec.rb +2 -1
- data/spec/integration/chunker_spec.rb +1 -1
- data/spec/integration/database.yml +10 -0
- data/spec/integration/entangler_spec.rb +3 -1
- data/spec/integration/integration_helper.rb +23 -5
- data/spec/integration/lhm_spec.rb +75 -0
- data/spec/integration/proxysql_spec.rb +34 -0
- data/spec/integration/sql_retry/db_connection_helper.rb +52 -0
- data/spec/integration/sql_retry/lock_wait_spec.rb +8 -6
- data/spec/integration/sql_retry/lock_wait_timeout_test_helper.rb +19 -9
- data/spec/integration/sql_retry/proxysql_helper.rb +22 -0
- data/spec/integration/sql_retry/retry_with_proxysql_spec.rb +108 -0
- data/spec/integration/toxiproxy_helper.rb +40 -0
- data/spec/test_helper.rb +21 -0
- data/spec/unit/chunk_insert_spec.rb +7 -2
- data/spec/unit/chunker_spec.rb +45 -42
- data/spec/unit/connection_spec.rb +22 -4
- data/spec/unit/entangler_spec.rb +41 -11
- data/spec/unit/throttler/slave_lag_spec.rb +13 -8
- metadata +76 -11
- data/gemfiles/ar-2.3_mysql.gemfile +0 -6
- data/gemfiles/ar-3.2_mysql.gemfile +0 -5
- data/gemfiles/ar-3.2_mysql2.gemfile +0 -5
- data/gemfiles/ar-4.0_mysql2.gemfile +0 -5
- data/gemfiles/ar-4.1_mysql2.gemfile +0 -5
- data/gemfiles/ar-4.2_mysql2.gemfile +0 -5
- data/gemfiles/ar-5.0_mysql2.gemfile +0 -5
data/spec/unit/chunker_spec.rb
CHANGED
@@ -12,11 +12,14 @@ require 'lhm/connection'
|
|
12
12
|
describe Lhm::Chunker do
|
13
13
|
include UnitHelper
|
14
14
|
|
15
|
+
EXPECTED_RETRY_FLAGS = {:should_retry => true, :retry_options => {}}
|
16
|
+
|
15
17
|
before(:each) do
|
16
18
|
@origin = Lhm::Table.new('foo')
|
17
19
|
@destination = Lhm::Table.new('bar')
|
18
20
|
@migration = Lhm::Migration.new(@origin, @destination)
|
19
21
|
@connection = mock()
|
22
|
+
@connection.stubs(:execute).returns([["dummy"]])
|
20
23
|
# This is a poor man's stub
|
21
24
|
@throttler = Object.new
|
22
25
|
def @throttler.run
|
@@ -38,11 +41,11 @@ describe Lhm::Chunker do
|
|
38
41
|
5
|
39
42
|
end
|
40
43
|
|
41
|
-
@connection.expects(:select_value).with(regexp_matches(/where id >= 1 order by id limit 1 offset 4/),
|
42
|
-
@connection.expects(:select_value).with(regexp_matches(/where id >= 8 order by id limit 1 offset 4/),
|
43
|
-
@connection.expects(:update).with(regexp_matches(/between 1 and 7/),
|
44
|
-
@connection.expects(:update).with(regexp_matches(/between 8 and 10/),
|
45
|
-
@connection.expects(:execute).twice.with(regexp_matches(/show warnings/),
|
44
|
+
@connection.expects(:select_value).with(regexp_matches(/where id >= 1 order by id limit 1 offset 4/),EXPECTED_RETRY_FLAGS).returns(7)
|
45
|
+
@connection.expects(:select_value).with(regexp_matches(/where id >= 8 order by id limit 1 offset 4/),EXPECTED_RETRY_FLAGS).returns(21)
|
46
|
+
@connection.expects(:update).with(regexp_matches(/between 1 and 7/),EXPECTED_RETRY_FLAGS).returns(2)
|
47
|
+
@connection.expects(:update).with(regexp_matches(/between 8 and 10/),EXPECTED_RETRY_FLAGS).returns(2)
|
48
|
+
@connection.expects(:execute).twice.with(regexp_matches(/show warnings/),EXPECTED_RETRY_FLAGS).returns([])
|
46
49
|
|
47
50
|
@chunker.run
|
48
51
|
end
|
@@ -53,17 +56,17 @@ describe Lhm::Chunker do
|
|
53
56
|
2
|
54
57
|
end
|
55
58
|
|
56
|
-
@connection.expects(:select_value).with(regexp_matches(/where id >= 1 order by id limit 1 offset 1/),
|
57
|
-
@connection.expects(:select_value).with(regexp_matches(/where id >= 3 order by id limit 1 offset 1/),
|
58
|
-
@connection.expects(:select_value).with(regexp_matches(/where id >= 5 order by id limit 1 offset 1/),
|
59
|
-
@connection.expects(:select_value).with(regexp_matches(/where id >= 7 order by id limit 1 offset 1/),
|
60
|
-
@connection.expects(:select_value).with(regexp_matches(/where id >= 9 order by id limit 1 offset 1/),
|
59
|
+
@connection.expects(:select_value).with(regexp_matches(/where id >= 1 order by id limit 1 offset 1/),EXPECTED_RETRY_FLAGS).returns(2)
|
60
|
+
@connection.expects(:select_value).with(regexp_matches(/where id >= 3 order by id limit 1 offset 1/),EXPECTED_RETRY_FLAGS).returns(4)
|
61
|
+
@connection.expects(:select_value).with(regexp_matches(/where id >= 5 order by id limit 1 offset 1/),EXPECTED_RETRY_FLAGS).returns(6)
|
62
|
+
@connection.expects(:select_value).with(regexp_matches(/where id >= 7 order by id limit 1 offset 1/),EXPECTED_RETRY_FLAGS).returns(8)
|
63
|
+
@connection.expects(:select_value).with(regexp_matches(/where id >= 9 order by id limit 1 offset 1/),EXPECTED_RETRY_FLAGS).returns(10)
|
61
64
|
|
62
|
-
@connection.expects(:update).with(regexp_matches(/between 1 and 2/),
|
63
|
-
@connection.expects(:update).with(regexp_matches(/between 3 and 4/),
|
64
|
-
@connection.expects(:update).with(regexp_matches(/between 5 and 6/),
|
65
|
-
@connection.expects(:update).with(regexp_matches(/between 7 and 8/),
|
66
|
-
@connection.expects(:update).with(regexp_matches(/between 9 and 10/),
|
65
|
+
@connection.expects(:update).with(regexp_matches(/between 1 and 2/),EXPECTED_RETRY_FLAGS).returns(2)
|
66
|
+
@connection.expects(:update).with(regexp_matches(/between 3 and 4/),EXPECTED_RETRY_FLAGS).returns(2)
|
67
|
+
@connection.expects(:update).with(regexp_matches(/between 5 and 6/),EXPECTED_RETRY_FLAGS).returns(2)
|
68
|
+
@connection.expects(:update).with(regexp_matches(/between 7 and 8/),EXPECTED_RETRY_FLAGS).returns(2)
|
69
|
+
@connection.expects(:update).with(regexp_matches(/between 9 and 10/),EXPECTED_RETRY_FLAGS).returns(2)
|
67
70
|
|
68
71
|
@chunker.run
|
69
72
|
end
|
@@ -80,17 +83,17 @@ describe Lhm::Chunker do
|
|
80
83
|
end
|
81
84
|
end
|
82
85
|
|
83
|
-
@connection.expects(:select_value).with(regexp_matches(/where id >= 1 order by id limit 1 offset 1/),
|
84
|
-
@connection.expects(:select_value).with(regexp_matches(/where id >= 3 order by id limit 1 offset 2/),
|
85
|
-
@connection.expects(:select_value).with(regexp_matches(/where id >= 6 order by id limit 1 offset 2/),
|
86
|
-
@connection.expects(:select_value).with(regexp_matches(/where id >= 9 order by id limit 1 offset 2/),
|
86
|
+
@connection.expects(:select_value).with(regexp_matches(/where id >= 1 order by id limit 1 offset 1/),EXPECTED_RETRY_FLAGS).returns(2)
|
87
|
+
@connection.expects(:select_value).with(regexp_matches(/where id >= 3 order by id limit 1 offset 2/),EXPECTED_RETRY_FLAGS).returns(5)
|
88
|
+
@connection.expects(:select_value).with(regexp_matches(/where id >= 6 order by id limit 1 offset 2/),EXPECTED_RETRY_FLAGS).returns(8)
|
89
|
+
@connection.expects(:select_value).with(regexp_matches(/where id >= 9 order by id limit 1 offset 2/),EXPECTED_RETRY_FLAGS).returns(nil)
|
87
90
|
|
88
|
-
@connection.expects(:update).with(regexp_matches(/between 1 and 2/),
|
89
|
-
@connection.expects(:update).with(regexp_matches(/between 3 and 5/),
|
90
|
-
@connection.expects(:update).with(regexp_matches(/between 6 and 8/),
|
91
|
-
@connection.expects(:update).with(regexp_matches(/between 9 and 10/),
|
91
|
+
@connection.expects(:update).with(regexp_matches(/between 1 and 2/),EXPECTED_RETRY_FLAGS).returns(2)
|
92
|
+
@connection.expects(:update).with(regexp_matches(/between 3 and 5/),EXPECTED_RETRY_FLAGS).returns(2)
|
93
|
+
@connection.expects(:update).with(regexp_matches(/between 6 and 8/),EXPECTED_RETRY_FLAGS).returns(2)
|
94
|
+
@connection.expects(:update).with(regexp_matches(/between 9 and 10/),EXPECTED_RETRY_FLAGS).returns(2)
|
92
95
|
|
93
|
-
@connection.expects(:execute).twice.with(regexp_matches(/show warnings/),
|
96
|
+
@connection.expects(:execute).twice.with(regexp_matches(/show warnings/),EXPECTED_RETRY_FLAGS).returns([])
|
94
97
|
|
95
98
|
@chunker.run
|
96
99
|
end
|
@@ -100,8 +103,8 @@ describe Lhm::Chunker do
|
|
100
103
|
:start => 1,
|
101
104
|
:limit => 1)
|
102
105
|
|
103
|
-
@connection.expects(:select_value).with(regexp_matches(/where id >= 1 order by id limit 1 offset 0/),
|
104
|
-
@connection.expects(:update).with(regexp_matches(/between 1 and 1/),
|
106
|
+
@connection.expects(:select_value).with(regexp_matches(/where id >= 1 order by id limit 1 offset 0/),EXPECTED_RETRY_FLAGS).returns(nil)
|
107
|
+
@connection.expects(:update).with(regexp_matches(/between 1 and 1/),EXPECTED_RETRY_FLAGS).returns(1)
|
105
108
|
|
106
109
|
@chunker.run
|
107
110
|
end
|
@@ -114,17 +117,17 @@ describe Lhm::Chunker do
|
|
114
117
|
2
|
115
118
|
end
|
116
119
|
|
117
|
-
@connection.expects(:select_value).with(regexp_matches(/where id >= 2 order by id limit 1 offset 1/),
|
118
|
-
@connection.expects(:select_value).with(regexp_matches(/where id >= 4 order by id limit 1 offset 1/),
|
119
|
-
@connection.expects(:select_value).with(regexp_matches(/where id >= 6 order by id limit 1 offset 1/),
|
120
|
-
@connection.expects(:select_value).with(regexp_matches(/where id >= 8 order by id limit 1 offset 1/),
|
121
|
-
@connection.expects(:select_value).with(regexp_matches(/where id >= 10 order by id limit 1 offset 1/),
|
120
|
+
@connection.expects(:select_value).with(regexp_matches(/where id >= 2 order by id limit 1 offset 1/),EXPECTED_RETRY_FLAGS).returns(3)
|
121
|
+
@connection.expects(:select_value).with(regexp_matches(/where id >= 4 order by id limit 1 offset 1/),EXPECTED_RETRY_FLAGS).returns(5)
|
122
|
+
@connection.expects(:select_value).with(regexp_matches(/where id >= 6 order by id limit 1 offset 1/),EXPECTED_RETRY_FLAGS).returns(7)
|
123
|
+
@connection.expects(:select_value).with(regexp_matches(/where id >= 8 order by id limit 1 offset 1/),EXPECTED_RETRY_FLAGS).returns(9)
|
124
|
+
@connection.expects(:select_value).with(regexp_matches(/where id >= 10 order by id limit 1 offset 1/),EXPECTED_RETRY_FLAGS).returns(nil)
|
122
125
|
|
123
|
-
@connection.expects(:update).with(regexp_matches(/between 2 and 3/),
|
124
|
-
@connection.expects(:update).with(regexp_matches(/between 4 and 5/),
|
125
|
-
@connection.expects(:update).with(regexp_matches(/between 6 and 7/),
|
126
|
-
@connection.expects(:update).with(regexp_matches(/between 8 and 9/),
|
127
|
-
@connection.expects(:update).with(regexp_matches(/between 10 and 10/),
|
126
|
+
@connection.expects(:update).with(regexp_matches(/between 2 and 3/),EXPECTED_RETRY_FLAGS).returns(2)
|
127
|
+
@connection.expects(:update).with(regexp_matches(/between 4 and 5/),EXPECTED_RETRY_FLAGS).returns(2)
|
128
|
+
@connection.expects(:update).with(regexp_matches(/between 6 and 7/),EXPECTED_RETRY_FLAGS).returns(2)
|
129
|
+
@connection.expects(:update).with(regexp_matches(/between 8 and 9/),EXPECTED_RETRY_FLAGS).returns(2)
|
130
|
+
@connection.expects(:update).with(regexp_matches(/between 10 and 10/),EXPECTED_RETRY_FLAGS).returns(1)
|
128
131
|
|
129
132
|
@chunker.run
|
130
133
|
end
|
@@ -138,9 +141,9 @@ describe Lhm::Chunker do
|
|
138
141
|
2
|
139
142
|
end
|
140
143
|
|
141
|
-
@connection.expects(:select_value).with(regexp_matches(/where id >= 1 order by id limit 1 offset 1/),
|
142
|
-
@connection.expects(:update).with(regexp_matches(/where \(foo.created_at > '2013-07-10' or foo.baz = 'quux'\) and `foo`/),
|
143
|
-
@connection.expects(:execute).with(regexp_matches(/show warnings/),
|
144
|
+
@connection.expects(:select_value).with(regexp_matches(/where id >= 1 order by id limit 1 offset 1/),EXPECTED_RETRY_FLAGS).returns(2)
|
145
|
+
@connection.expects(:update).with(regexp_matches(/where \(foo.created_at > '2013-07-10' or foo.baz = 'quux'\) and `foo`/),EXPECTED_RETRY_FLAGS).returns(1)
|
146
|
+
@connection.expects(:execute).with(regexp_matches(/show warnings/),EXPECTED_RETRY_FLAGS).returns([])
|
144
147
|
|
145
148
|
def @migration.conditions
|
146
149
|
"where foo.created_at > '2013-07-10' or foo.baz = 'quux'"
|
@@ -158,9 +161,9 @@ describe Lhm::Chunker do
|
|
158
161
|
2
|
159
162
|
end
|
160
163
|
|
161
|
-
@connection.expects(:select_value).with(regexp_matches(/where id >= 1 order by id limit 1 offset 1/),
|
162
|
-
@connection.expects(:update).with(regexp_matches(/inner join bar on foo.id = bar.foo_id and/),
|
163
|
-
@connection.expects(:execute).with(regexp_matches(/show warnings/),
|
164
|
+
@connection.expects(:select_value).with(regexp_matches(/where id >= 1 order by id limit 1 offset 1/),EXPECTED_RETRY_FLAGS).returns(2)
|
165
|
+
@connection.expects(:update).with(regexp_matches(/inner join bar on foo.id = bar.foo_id and/),EXPECTED_RETRY_FLAGS).returns(1)
|
166
|
+
@connection.expects(:execute).with(regexp_matches(/show warnings/),EXPECTED_RETRY_FLAGS).returns([])
|
164
167
|
|
165
168
|
def @migration.conditions
|
166
169
|
'inner join bar on foo.id = bar.foo_id'
|
@@ -1,4 +1,5 @@
|
|
1
1
|
require 'lhm/connection'
|
2
|
+
require 'lhm/proxysql_helper'
|
2
3
|
|
3
4
|
describe Lhm::Connection do
|
4
5
|
|
@@ -12,10 +13,11 @@ describe Lhm::Connection do
|
|
12
13
|
it "Should find use calling file as prefix" do
|
13
14
|
ar_connection = mock()
|
14
15
|
ar_connection.stubs(:execute).raises(LOCK_WAIT).then.returns(true)
|
16
|
+
ar_connection.stubs(:active?).returns(true)
|
15
17
|
|
16
18
|
connection = Lhm::Connection.new(connection: ar_connection)
|
17
19
|
|
18
|
-
connection.execute("SHOW TABLES", { base_interval: 0 })
|
20
|
+
connection.execute("SHOW TABLES", should_retry: true, retry_options: { base_interval: 0 })
|
19
21
|
|
20
22
|
log_messages = @logs.string.split("\n")
|
21
23
|
assert_equal(1, log_messages.length)
|
@@ -27,10 +29,11 @@ describe Lhm::Connection do
|
|
27
29
|
ar_connection.stubs(:execute).raises(LOCK_WAIT)
|
28
30
|
.then.raises(LOCK_WAIT)
|
29
31
|
.then.returns(true)
|
32
|
+
ar_connection.stubs(:active?).returns(true)
|
30
33
|
|
31
34
|
connection = Lhm::Connection.new(connection: ar_connection)
|
32
35
|
|
33
|
-
connection.execute("SHOW TABLES", { base_interval: 0, tries: 3 })
|
36
|
+
connection.execute("SHOW TABLES", should_retry: true, retry_options: { base_interval: 0, tries: 3 })
|
34
37
|
|
35
38
|
log_messages = @logs.string.split("\n")
|
36
39
|
assert_equal(2, log_messages.length)
|
@@ -41,10 +44,11 @@ describe Lhm::Connection do
|
|
41
44
|
ar_connection.stubs(:update).raises(LOCK_WAIT)
|
42
45
|
.then.raises(LOCK_WAIT)
|
43
46
|
.then.returns(1)
|
47
|
+
ar_connection.stubs(:active?).returns(true)
|
44
48
|
|
45
49
|
connection = Lhm::Connection.new(connection: ar_connection)
|
46
50
|
|
47
|
-
val = connection.update("SHOW TABLES", { base_interval: 0, tries: 3 })
|
51
|
+
val = connection.update("SHOW TABLES", should_retry: true, retry_options:{ base_interval: 0, tries: 3 })
|
48
52
|
|
49
53
|
log_messages = @logs.string.split("\n")
|
50
54
|
assert_equal val, 1
|
@@ -56,13 +60,27 @@ describe Lhm::Connection do
|
|
56
60
|
ar_connection.stubs(:select_value).raises(LOCK_WAIT)
|
57
61
|
.then.raises(LOCK_WAIT)
|
58
62
|
.then.returns("dummy")
|
63
|
+
ar_connection.stubs(:active?).returns(true)
|
59
64
|
|
60
65
|
connection = Lhm::Connection.new(connection: ar_connection)
|
61
66
|
|
62
|
-
val = connection.select_value("SHOW TABLES", { base_interval: 0, tries: 3 })
|
67
|
+
val = connection.select_value("SHOW TABLES", should_retry: true, retry_options: { base_interval: 0, tries: 3 })
|
63
68
|
|
64
69
|
log_messages = @logs.string.split("\n")
|
65
70
|
assert_equal val, "dummy"
|
66
71
|
assert_equal(2, log_messages.length)
|
67
72
|
end
|
73
|
+
|
74
|
+
it "Queries should be tagged with ProxySQL tag if requested" do
|
75
|
+
ar_connection = mock()
|
76
|
+
ar_connection.expects(:public_send).with(:select_value, "#{Lhm::ProxySQLHelper::ANNOTATION}SHOW TABLES").returns("dummy")
|
77
|
+
ar_connection.stubs(:execute).times(4).returns([["dummy"]])
|
78
|
+
ar_connection.stubs(:active?).returns(true)
|
79
|
+
|
80
|
+
connection = Lhm::Connection.new(connection: ar_connection, options: { reconnect_with_consistent_host: true })
|
81
|
+
|
82
|
+
val = connection.select_value("SHOW TABLES", should_retry: true, retry_options: { base_interval: 0, tries: 3 })
|
83
|
+
|
84
|
+
assert_equal val, "dummy"
|
85
|
+
end
|
68
86
|
end
|
data/spec/unit/entangler_spec.rb
CHANGED
@@ -63,9 +63,13 @@ describe Lhm::Entangler do
|
|
63
63
|
it 'should retry trigger creation when it hits a lock wait timeout' do
|
64
64
|
tries = 1
|
65
65
|
ar_connection = mock()
|
66
|
-
ar_connection.
|
66
|
+
ar_connection.stubs(:execute)
|
67
|
+
.returns([["dummy"]], [["dummy"]], [["dummy"]])
|
68
|
+
.then
|
69
|
+
.raises(Mysql2::Error, 'Lock wait timeout exceeded; try restarting transaction')
|
70
|
+
ar_connection.stubs(:active?).returns(true)
|
67
71
|
|
68
|
-
connection = Lhm::Connection.new(connection: ar_connection)
|
72
|
+
connection = Lhm::Connection.new(connection: ar_connection, options: {reconnect_with_consistent_host: true})
|
69
73
|
|
70
74
|
@entangler = Lhm::Entangler.new(@migration, connection, retriable: {base_interval: 0, tries: tries})
|
71
75
|
|
@@ -74,18 +78,28 @@ describe Lhm::Entangler do
|
|
74
78
|
|
75
79
|
it 'should not retry trigger creation with other mysql errors' do
|
76
80
|
ar_connection = mock()
|
77
|
-
ar_connection.
|
78
|
-
|
79
|
-
|
80
|
-
|
81
|
+
ar_connection.stubs(:execute)
|
82
|
+
.returns([["dummy"]], [["dummy"]], [["dummy"]])
|
83
|
+
.then
|
84
|
+
.raises(Mysql2::Error, 'The MySQL server is running with the --read-only option so it cannot execute this statement.')
|
85
|
+
ar_connection.stubs(:active?).returns(true)
|
86
|
+
connection = Lhm::Connection.new(connection: ar_connection, options: {reconnect_with_consistent_host: true})
|
87
|
+
|
88
|
+
@entangler = Lhm::Entangler.new(@migration, connection, retriable: { base_interval: 0 })
|
81
89
|
assert_raises(Mysql2::Error) { @entangler.before }
|
82
90
|
end
|
83
91
|
|
84
92
|
it 'should succesfully finish after retrying' do
|
85
93
|
ar_connection = mock()
|
86
|
-
ar_connection.stubs(:execute)
|
94
|
+
ar_connection.stubs(:execute)
|
95
|
+
.returns([["dummy"]], [["dummy"]], [["dummy"]])
|
96
|
+
.then
|
97
|
+
.raises(Mysql2::Error, 'Lock wait timeout exceeded; try restarting transaction')
|
98
|
+
.then
|
99
|
+
.returns([["dummy"]])
|
100
|
+
ar_connection.stubs(:active?).returns(true)
|
87
101
|
|
88
|
-
connection = Lhm::Connection.new(connection: ar_connection)
|
102
|
+
connection = Lhm::Connection.new(connection: ar_connection, options: {reconnect_with_consistent_host: true})
|
89
103
|
|
90
104
|
@entangler = Lhm::Entangler.new(@migration, connection, retriable: {base_interval: 0})
|
91
105
|
|
@@ -94,9 +108,25 @@ describe Lhm::Entangler do
|
|
94
108
|
|
95
109
|
it 'should retry as many times as specified by configuration' do
|
96
110
|
ar_connection = mock()
|
97
|
-
ar_connection.
|
98
|
-
|
99
|
-
|
111
|
+
ar_connection.stubs(:execute)
|
112
|
+
.returns([["dummy"]], [["dummy"]], [["dummy"]]) # initial
|
113
|
+
.then
|
114
|
+
.raises(Mysql2::Error, 'Lock wait timeout exceeded; try restarting transaction')
|
115
|
+
.then
|
116
|
+
.returns([["dummy"]]) # reconnect 1
|
117
|
+
.then
|
118
|
+
.raises(Mysql2::Error, 'Lock wait timeout exceeded; try restarting transaction')
|
119
|
+
.then
|
120
|
+
.returns([["dummy"]]) # reconnect 2
|
121
|
+
.then
|
122
|
+
.raises(Mysql2::Error, 'Lock wait timeout exceeded; try restarting transaction')
|
123
|
+
.then
|
124
|
+
.returns([["dummy"]]) # reconnect 3
|
125
|
+
.then
|
126
|
+
.raises(Mysql2::Error, 'Lock wait timeout exceeded; try restarting transaction') # final error
|
127
|
+
ar_connection.stubs(:active?).returns(true)
|
128
|
+
|
129
|
+
connection = Lhm::Connection.new(connection: ar_connection, options: {reconnect_with_consistent_host: true})
|
100
130
|
|
101
131
|
@entangler = Lhm::Entangler.new(@migration, connection, retriable: {tries: 5, base_interval: 0})
|
102
132
|
|
@@ -39,7 +39,7 @@ describe Lhm::Throttler::Slave do
|
|
39
39
|
@logs = StringIO.new
|
40
40
|
Lhm.logger = Logger.new(@logs)
|
41
41
|
|
42
|
-
@dummy_mysql_client_config = lambda { {'username' => 'user', 'password' => 'pw', 'database' => 'db'} }
|
42
|
+
@dummy_mysql_client_config = lambda { { 'username' => 'user', 'password' => 'pw', 'database' => 'db' } }
|
43
43
|
end
|
44
44
|
|
45
45
|
describe "#client" do
|
@@ -64,7 +64,7 @@ describe Lhm::Throttler::Slave do
|
|
64
64
|
|
65
65
|
describe 'with proper config' do
|
66
66
|
it "creates a new Mysql2::Client" do
|
67
|
-
expected_config = {username: 'user', password: 'pw', database: 'db', host: 'slave'}
|
67
|
+
expected_config = { username: 'user', password: 'pw', database: 'db', host: 'slave' }
|
68
68
|
Mysql2::Client.stubs(:new).with(expected_config).returns(mock())
|
69
69
|
|
70
70
|
assert Lhm::Throttler::Slave.new('slave', @dummy_mysql_client_config).connection
|
@@ -73,8 +73,12 @@ describe Lhm::Throttler::Slave do
|
|
73
73
|
|
74
74
|
describe 'with active record config' do
|
75
75
|
it 'logs and creates client' do
|
76
|
-
active_record_config = {username: 'user', password: 'pw', database: 'db'}
|
77
|
-
ActiveRecord::
|
76
|
+
active_record_config = { username: 'user', password: 'pw', database: 'db' }
|
77
|
+
if ActiveRecord::VERSION::MAJOR > 6 || ActiveRecord::VERSION::MAJOR == 6 && ActiveRecord::VERSION::MINOR >= 1
|
78
|
+
ActiveRecord::Base.stubs(:connection_pool).returns(stub(db_config: stub(configuration_hash: active_record_config)))
|
79
|
+
else
|
80
|
+
ActiveRecord::Base.stubs(:connection_pool).returns(stub(spec: stub(config: active_record_config)))
|
81
|
+
end
|
78
82
|
|
79
83
|
Mysql2::Client.stubs(:new).returns(mock())
|
80
84
|
|
@@ -92,9 +96,9 @@ describe Lhm::Throttler::Slave do
|
|
92
96
|
class Connection
|
93
97
|
def self.query(query)
|
94
98
|
if query == Lhm::Throttler::Slave::SQL_SELECT_MAX_SLAVE_LAG
|
95
|
-
[{'Seconds_Behind_Master' => 20}]
|
99
|
+
[{ 'Seconds_Behind_Master' => 20 }]
|
96
100
|
elsif query == Lhm::Throttler::Slave::SQL_SELECT_SLAVE_HOSTS
|
97
|
-
[{'host' => '1.1.1.1:80'}]
|
101
|
+
[{ 'host' => '1.1.1.1:80' }]
|
98
102
|
end
|
99
103
|
end
|
100
104
|
end
|
@@ -104,7 +108,7 @@ describe Lhm::Throttler::Slave do
|
|
104
108
|
|
105
109
|
class StoppedConnection
|
106
110
|
def self.query(query)
|
107
|
-
[{'Seconds_Behind_Master' => nil}]
|
111
|
+
[{ 'Seconds_Behind_Master' => nil }]
|
108
112
|
end
|
109
113
|
end
|
110
114
|
|
@@ -286,7 +290,7 @@ describe Lhm::Throttler::SlaveLag do
|
|
286
290
|
describe 'with the :check_only option' do
|
287
291
|
describe 'with a callable argument' do
|
288
292
|
before do
|
289
|
-
check_only = lambda {{'host' => '1.1.1.3'}}
|
293
|
+
check_only = lambda { { 'host' => '1.1.1.3' } }
|
290
294
|
@throttler = Lhm::Throttler::SlaveLag.new :check_only => check_only
|
291
295
|
end
|
292
296
|
|
@@ -300,6 +304,7 @@ describe Lhm::Throttler::SlaveLag do
|
|
300
304
|
describe 'with a non-callable argument' do
|
301
305
|
before do
|
302
306
|
@throttler = Lhm::Throttler::SlaveLag.new :check_only => 'I cannot be called'
|
307
|
+
|
303
308
|
def @throttler.master_slave_hosts
|
304
309
|
['1.1.1.1', '1.1.1.4']
|
305
310
|
end
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: lhm-shopify
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 3.5.
|
4
|
+
version: 3.5.1
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- SoundCloud
|
@@ -12,7 +12,7 @@ authors:
|
|
12
12
|
autorequire:
|
13
13
|
bindir: bin
|
14
14
|
cert_chain: []
|
15
|
-
date: 2021-12-
|
15
|
+
date: 2021-12-06 00:00:00.000000000 Z
|
16
16
|
dependencies:
|
17
17
|
- !ruby/object:Gem::Dependency
|
18
18
|
name: retriable
|
@@ -28,6 +28,20 @@ dependencies:
|
|
28
28
|
- - ">="
|
29
29
|
- !ruby/object:Gem::Version
|
30
30
|
version: 3.0.0
|
31
|
+
- !ruby/object:Gem::Dependency
|
32
|
+
name: activerecord
|
33
|
+
requirement: !ruby/object:Gem::Requirement
|
34
|
+
requirements:
|
35
|
+
- - ">="
|
36
|
+
- !ruby/object:Gem::Version
|
37
|
+
version: '0'
|
38
|
+
type: :development
|
39
|
+
prerelease: false
|
40
|
+
version_requirements: !ruby/object:Gem::Requirement
|
41
|
+
requirements:
|
42
|
+
- - ">="
|
43
|
+
- !ruby/object:Gem::Version
|
44
|
+
version: '0'
|
31
45
|
- !ruby/object:Gem::Dependency
|
32
46
|
name: minitest
|
33
47
|
requirement: !ruby/object:Gem::Requirement
|
@@ -57,7 +71,7 @@ dependencies:
|
|
57
71
|
- !ruby/object:Gem::Version
|
58
72
|
version: '0'
|
59
73
|
- !ruby/object:Gem::Dependency
|
60
|
-
name:
|
74
|
+
name: after_do
|
61
75
|
requirement: !ruby/object:Gem::Requirement
|
62
76
|
requirements:
|
63
77
|
- - ">="
|
@@ -71,7 +85,7 @@ dependencies:
|
|
71
85
|
- !ruby/object:Gem::Version
|
72
86
|
version: '0'
|
73
87
|
- !ruby/object:Gem::Dependency
|
74
|
-
name:
|
88
|
+
name: rake
|
75
89
|
requirement: !ruby/object:Gem::Requirement
|
76
90
|
requirements:
|
77
91
|
- - ">="
|
@@ -112,6 +126,48 @@ dependencies:
|
|
112
126
|
- - ">="
|
113
127
|
- !ruby/object:Gem::Version
|
114
128
|
version: '0'
|
129
|
+
- !ruby/object:Gem::Dependency
|
130
|
+
name: toxiproxy
|
131
|
+
requirement: !ruby/object:Gem::Requirement
|
132
|
+
requirements:
|
133
|
+
- - ">="
|
134
|
+
- !ruby/object:Gem::Version
|
135
|
+
version: '0'
|
136
|
+
type: :development
|
137
|
+
prerelease: false
|
138
|
+
version_requirements: !ruby/object:Gem::Requirement
|
139
|
+
requirements:
|
140
|
+
- - ">="
|
141
|
+
- !ruby/object:Gem::Version
|
142
|
+
version: '0'
|
143
|
+
- !ruby/object:Gem::Dependency
|
144
|
+
name: appraisal
|
145
|
+
requirement: !ruby/object:Gem::Requirement
|
146
|
+
requirements:
|
147
|
+
- - ">="
|
148
|
+
- !ruby/object:Gem::Version
|
149
|
+
version: '0'
|
150
|
+
type: :development
|
151
|
+
prerelease: false
|
152
|
+
version_requirements: !ruby/object:Gem::Requirement
|
153
|
+
requirements:
|
154
|
+
- - ">="
|
155
|
+
- !ruby/object:Gem::Version
|
156
|
+
version: '0'
|
157
|
+
- !ruby/object:Gem::Dependency
|
158
|
+
name: byebug
|
159
|
+
requirement: !ruby/object:Gem::Requirement
|
160
|
+
requirements:
|
161
|
+
- - ">="
|
162
|
+
- !ruby/object:Gem::Version
|
163
|
+
version: '0'
|
164
|
+
type: :development
|
165
|
+
prerelease: false
|
166
|
+
version_requirements: !ruby/object:Gem::Requirement
|
167
|
+
requirements:
|
168
|
+
- - ">="
|
169
|
+
- !ruby/object:Gem::Version
|
170
|
+
version: '0'
|
115
171
|
description: Migrate large tables without downtime by copying to a temporary table
|
116
172
|
in chunks. The old table is not dropped. Instead, it is moved to timestamp_table_name
|
117
173
|
for verification.
|
@@ -124,20 +180,23 @@ files:
|
|
124
180
|
- ".gitignore"
|
125
181
|
- ".rubocop.yml"
|
126
182
|
- ".travis.yml"
|
183
|
+
- Appraisals
|
127
184
|
- CHANGELOG.md
|
128
185
|
- Gemfile
|
186
|
+
- Gemfile.lock
|
129
187
|
- LICENSE
|
130
188
|
- README.md
|
131
189
|
- Rakefile
|
132
190
|
- dev.yml
|
133
191
|
- docker-compose.yml
|
134
|
-
- gemfiles/
|
135
|
-
- gemfiles/
|
136
|
-
- gemfiles/
|
137
|
-
- gemfiles/
|
138
|
-
- gemfiles/
|
139
|
-
- gemfiles/
|
140
|
-
- gemfiles/
|
192
|
+
- gemfiles/activerecord_5.2.gemfile
|
193
|
+
- gemfiles/activerecord_5.2.gemfile.lock
|
194
|
+
- gemfiles/activerecord_6.0.gemfile
|
195
|
+
- gemfiles/activerecord_6.0.gemfile.lock
|
196
|
+
- gemfiles/activerecord_6.1.gemfile
|
197
|
+
- gemfiles/activerecord_6.1.gemfile.lock
|
198
|
+
- gemfiles/activerecord_7.0.0.alpha2.gemfile
|
199
|
+
- gemfiles/activerecord_7.0.0.alpha2.gemfile.lock
|
141
200
|
- lhm.gemspec
|
142
201
|
- lib/lhm-shopify.rb
|
143
202
|
- lib/lhm.rb
|
@@ -155,6 +214,7 @@ files:
|
|
155
214
|
- lib/lhm/migration.rb
|
156
215
|
- lib/lhm/migrator.rb
|
157
216
|
- lib/lhm/printer.rb
|
217
|
+
- lib/lhm/proxysql_helper.rb
|
158
218
|
- lib/lhm/railtie.rb
|
159
219
|
- lib/lhm/sql_helper.rb
|
160
220
|
- lib/lhm/sql_retry.rb
|
@@ -199,9 +259,14 @@ files:
|
|
199
259
|
- spec/integration/lhm_spec.rb
|
200
260
|
- spec/integration/lock_wait_timeout_spec.rb
|
201
261
|
- spec/integration/locked_switcher_spec.rb
|
262
|
+
- spec/integration/proxysql_spec.rb
|
263
|
+
- spec/integration/sql_retry/db_connection_helper.rb
|
202
264
|
- spec/integration/sql_retry/lock_wait_spec.rb
|
203
265
|
- spec/integration/sql_retry/lock_wait_timeout_test_helper.rb
|
266
|
+
- spec/integration/sql_retry/proxysql_helper.rb
|
267
|
+
- spec/integration/sql_retry/retry_with_proxysql_spec.rb
|
204
268
|
- spec/integration/table_spec.rb
|
269
|
+
- spec/integration/toxiproxy_helper.rb
|
205
270
|
- spec/test_helper.rb
|
206
271
|
- spec/unit/atomic_switcher_spec.rb
|
207
272
|
- spec/unit/chunk_finder_spec.rb
|