lhm-shopify 3.5.0 → 3.5.1

Sign up to get free protection for your applications and to get access to all the features.
Files changed (59) hide show
  1. checksums.yaml +4 -4
  2. data/.github/workflows/test.yml +17 -4
  3. data/.gitignore +0 -2
  4. data/Appraisals +24 -0
  5. data/Gemfile.lock +66 -0
  6. data/README.md +42 -0
  7. data/Rakefile +1 -0
  8. data/dev.yml +18 -3
  9. data/docker-compose.yml +15 -3
  10. data/gemfiles/activerecord_5.2.gemfile +9 -0
  11. data/gemfiles/activerecord_5.2.gemfile.lock +65 -0
  12. data/gemfiles/activerecord_6.0.gemfile +7 -0
  13. data/gemfiles/activerecord_6.0.gemfile.lock +67 -0
  14. data/gemfiles/activerecord_6.1.gemfile +7 -0
  15. data/gemfiles/activerecord_6.1.gemfile.lock +66 -0
  16. data/gemfiles/activerecord_7.0.0.alpha2.gemfile +7 -0
  17. data/gemfiles/activerecord_7.0.0.alpha2.gemfile.lock +64 -0
  18. data/lhm.gemspec +7 -3
  19. data/lib/lhm/atomic_switcher.rb +1 -1
  20. data/lib/lhm/chunk_insert.rb +2 -1
  21. data/lib/lhm/chunker.rb +3 -3
  22. data/lib/lhm/cleanup/current.rb +1 -1
  23. data/lib/lhm/connection.rb +50 -11
  24. data/lib/lhm/entangler.rb +2 -2
  25. data/lib/lhm/invoker.rb +2 -2
  26. data/lib/lhm/proxysql_helper.rb +10 -0
  27. data/lib/lhm/sql_retry.rb +126 -8
  28. data/lib/lhm/throttler/slave_lag.rb +19 -2
  29. data/lib/lhm/version.rb +1 -1
  30. data/lib/lhm.rb +22 -8
  31. data/scripts/mysql/writer/create_users.sql +3 -0
  32. data/spec/integration/atomic_switcher_spec.rb +27 -10
  33. data/spec/integration/chunk_insert_spec.rb +2 -1
  34. data/spec/integration/chunker_spec.rb +1 -1
  35. data/spec/integration/database.yml +10 -0
  36. data/spec/integration/entangler_spec.rb +3 -1
  37. data/spec/integration/integration_helper.rb +23 -5
  38. data/spec/integration/lhm_spec.rb +75 -0
  39. data/spec/integration/proxysql_spec.rb +34 -0
  40. data/spec/integration/sql_retry/db_connection_helper.rb +52 -0
  41. data/spec/integration/sql_retry/lock_wait_spec.rb +8 -6
  42. data/spec/integration/sql_retry/lock_wait_timeout_test_helper.rb +19 -9
  43. data/spec/integration/sql_retry/proxysql_helper.rb +22 -0
  44. data/spec/integration/sql_retry/retry_with_proxysql_spec.rb +108 -0
  45. data/spec/integration/toxiproxy_helper.rb +40 -0
  46. data/spec/test_helper.rb +21 -0
  47. data/spec/unit/chunk_insert_spec.rb +7 -2
  48. data/spec/unit/chunker_spec.rb +45 -42
  49. data/spec/unit/connection_spec.rb +22 -4
  50. data/spec/unit/entangler_spec.rb +41 -11
  51. data/spec/unit/throttler/slave_lag_spec.rb +13 -8
  52. metadata +76 -11
  53. data/gemfiles/ar-2.3_mysql.gemfile +0 -6
  54. data/gemfiles/ar-3.2_mysql.gemfile +0 -5
  55. data/gemfiles/ar-3.2_mysql2.gemfile +0 -5
  56. data/gemfiles/ar-4.0_mysql2.gemfile +0 -5
  57. data/gemfiles/ar-4.1_mysql2.gemfile +0 -5
  58. data/gemfiles/ar-4.2_mysql2.gemfile +0 -5
  59. data/gemfiles/ar-5.0_mysql2.gemfile +0 -5
@@ -12,11 +12,14 @@ require 'lhm/connection'
12
12
  describe Lhm::Chunker do
13
13
  include UnitHelper
14
14
 
15
+ EXPECTED_RETRY_FLAGS = {:should_retry => true, :retry_options => {}}
16
+
15
17
  before(:each) do
16
18
  @origin = Lhm::Table.new('foo')
17
19
  @destination = Lhm::Table.new('bar')
18
20
  @migration = Lhm::Migration.new(@origin, @destination)
19
21
  @connection = mock()
22
+ @connection.stubs(:execute).returns([["dummy"]])
20
23
  # This is a poor man's stub
21
24
  @throttler = Object.new
22
25
  def @throttler.run
@@ -38,11 +41,11 @@ describe Lhm::Chunker do
38
41
  5
39
42
  end
40
43
 
41
- @connection.expects(:select_value).with(regexp_matches(/where id >= 1 order by id limit 1 offset 4/),{}).returns(7)
42
- @connection.expects(:select_value).with(regexp_matches(/where id >= 8 order by id limit 1 offset 4/),{}).returns(21)
43
- @connection.expects(:update).with(regexp_matches(/between 1 and 7/),{}).returns(2)
44
- @connection.expects(:update).with(regexp_matches(/between 8 and 10/),{}).returns(2)
45
- @connection.expects(:execute).twice.with(regexp_matches(/show warnings/),{}).returns([])
44
+ @connection.expects(:select_value).with(regexp_matches(/where id >= 1 order by id limit 1 offset 4/),EXPECTED_RETRY_FLAGS).returns(7)
45
+ @connection.expects(:select_value).with(regexp_matches(/where id >= 8 order by id limit 1 offset 4/),EXPECTED_RETRY_FLAGS).returns(21)
46
+ @connection.expects(:update).with(regexp_matches(/between 1 and 7/),EXPECTED_RETRY_FLAGS).returns(2)
47
+ @connection.expects(:update).with(regexp_matches(/between 8 and 10/),EXPECTED_RETRY_FLAGS).returns(2)
48
+ @connection.expects(:execute).twice.with(regexp_matches(/show warnings/),EXPECTED_RETRY_FLAGS).returns([])
46
49
 
47
50
  @chunker.run
48
51
  end
@@ -53,17 +56,17 @@ describe Lhm::Chunker do
53
56
  2
54
57
  end
55
58
 
56
- @connection.expects(:select_value).with(regexp_matches(/where id >= 1 order by id limit 1 offset 1/),{}).returns(2)
57
- @connection.expects(:select_value).with(regexp_matches(/where id >= 3 order by id limit 1 offset 1/),{}).returns(4)
58
- @connection.expects(:select_value).with(regexp_matches(/where id >= 5 order by id limit 1 offset 1/),{}).returns(6)
59
- @connection.expects(:select_value).with(regexp_matches(/where id >= 7 order by id limit 1 offset 1/),{}).returns(8)
60
- @connection.expects(:select_value).with(regexp_matches(/where id >= 9 order by id limit 1 offset 1/),{}).returns(10)
59
+ @connection.expects(:select_value).with(regexp_matches(/where id >= 1 order by id limit 1 offset 1/),EXPECTED_RETRY_FLAGS).returns(2)
60
+ @connection.expects(:select_value).with(regexp_matches(/where id >= 3 order by id limit 1 offset 1/),EXPECTED_RETRY_FLAGS).returns(4)
61
+ @connection.expects(:select_value).with(regexp_matches(/where id >= 5 order by id limit 1 offset 1/),EXPECTED_RETRY_FLAGS).returns(6)
62
+ @connection.expects(:select_value).with(regexp_matches(/where id >= 7 order by id limit 1 offset 1/),EXPECTED_RETRY_FLAGS).returns(8)
63
+ @connection.expects(:select_value).with(regexp_matches(/where id >= 9 order by id limit 1 offset 1/),EXPECTED_RETRY_FLAGS).returns(10)
61
64
 
62
- @connection.expects(:update).with(regexp_matches(/between 1 and 2/),{}).returns(2)
63
- @connection.expects(:update).with(regexp_matches(/between 3 and 4/),{}).returns(2)
64
- @connection.expects(:update).with(regexp_matches(/between 5 and 6/),{}).returns(2)
65
- @connection.expects(:update).with(regexp_matches(/between 7 and 8/),{}).returns(2)
66
- @connection.expects(:update).with(regexp_matches(/between 9 and 10/),{}).returns(2)
65
+ @connection.expects(:update).with(regexp_matches(/between 1 and 2/),EXPECTED_RETRY_FLAGS).returns(2)
66
+ @connection.expects(:update).with(regexp_matches(/between 3 and 4/),EXPECTED_RETRY_FLAGS).returns(2)
67
+ @connection.expects(:update).with(regexp_matches(/between 5 and 6/),EXPECTED_RETRY_FLAGS).returns(2)
68
+ @connection.expects(:update).with(regexp_matches(/between 7 and 8/),EXPECTED_RETRY_FLAGS).returns(2)
69
+ @connection.expects(:update).with(regexp_matches(/between 9 and 10/),EXPECTED_RETRY_FLAGS).returns(2)
67
70
 
68
71
  @chunker.run
69
72
  end
@@ -80,17 +83,17 @@ describe Lhm::Chunker do
80
83
  end
81
84
  end
82
85
 
83
- @connection.expects(:select_value).with(regexp_matches(/where id >= 1 order by id limit 1 offset 1/),{}).returns(2)
84
- @connection.expects(:select_value).with(regexp_matches(/where id >= 3 order by id limit 1 offset 2/),{}).returns(5)
85
- @connection.expects(:select_value).with(regexp_matches(/where id >= 6 order by id limit 1 offset 2/),{}).returns(8)
86
- @connection.expects(:select_value).with(regexp_matches(/where id >= 9 order by id limit 1 offset 2/),{}).returns(nil)
86
+ @connection.expects(:select_value).with(regexp_matches(/where id >= 1 order by id limit 1 offset 1/),EXPECTED_RETRY_FLAGS).returns(2)
87
+ @connection.expects(:select_value).with(regexp_matches(/where id >= 3 order by id limit 1 offset 2/),EXPECTED_RETRY_FLAGS).returns(5)
88
+ @connection.expects(:select_value).with(regexp_matches(/where id >= 6 order by id limit 1 offset 2/),EXPECTED_RETRY_FLAGS).returns(8)
89
+ @connection.expects(:select_value).with(regexp_matches(/where id >= 9 order by id limit 1 offset 2/),EXPECTED_RETRY_FLAGS).returns(nil)
87
90
 
88
- @connection.expects(:update).with(regexp_matches(/between 1 and 2/),{}).returns(2)
89
- @connection.expects(:update).with(regexp_matches(/between 3 and 5/),{}).returns(2)
90
- @connection.expects(:update).with(regexp_matches(/between 6 and 8/),{}).returns(2)
91
- @connection.expects(:update).with(regexp_matches(/between 9 and 10/),{}).returns(2)
91
+ @connection.expects(:update).with(regexp_matches(/between 1 and 2/),EXPECTED_RETRY_FLAGS).returns(2)
92
+ @connection.expects(:update).with(regexp_matches(/between 3 and 5/),EXPECTED_RETRY_FLAGS).returns(2)
93
+ @connection.expects(:update).with(regexp_matches(/between 6 and 8/),EXPECTED_RETRY_FLAGS).returns(2)
94
+ @connection.expects(:update).with(regexp_matches(/between 9 and 10/),EXPECTED_RETRY_FLAGS).returns(2)
92
95
 
93
- @connection.expects(:execute).twice.with(regexp_matches(/show warnings/),{}).returns([])
96
+ @connection.expects(:execute).twice.with(regexp_matches(/show warnings/),EXPECTED_RETRY_FLAGS).returns([])
94
97
 
95
98
  @chunker.run
96
99
  end
@@ -100,8 +103,8 @@ describe Lhm::Chunker do
100
103
  :start => 1,
101
104
  :limit => 1)
102
105
 
103
- @connection.expects(:select_value).with(regexp_matches(/where id >= 1 order by id limit 1 offset 0/),{}).returns(nil)
104
- @connection.expects(:update).with(regexp_matches(/between 1 and 1/),{}).returns(1)
106
+ @connection.expects(:select_value).with(regexp_matches(/where id >= 1 order by id limit 1 offset 0/),EXPECTED_RETRY_FLAGS).returns(nil)
107
+ @connection.expects(:update).with(regexp_matches(/between 1 and 1/),EXPECTED_RETRY_FLAGS).returns(1)
105
108
 
106
109
  @chunker.run
107
110
  end
@@ -114,17 +117,17 @@ describe Lhm::Chunker do
114
117
  2
115
118
  end
116
119
 
117
- @connection.expects(:select_value).with(regexp_matches(/where id >= 2 order by id limit 1 offset 1/),{}).returns(3)
118
- @connection.expects(:select_value).with(regexp_matches(/where id >= 4 order by id limit 1 offset 1/),{}).returns(5)
119
- @connection.expects(:select_value).with(regexp_matches(/where id >= 6 order by id limit 1 offset 1/),{}).returns(7)
120
- @connection.expects(:select_value).with(regexp_matches(/where id >= 8 order by id limit 1 offset 1/),{}).returns(9)
121
- @connection.expects(:select_value).with(regexp_matches(/where id >= 10 order by id limit 1 offset 1/),{}).returns(nil)
120
+ @connection.expects(:select_value).with(regexp_matches(/where id >= 2 order by id limit 1 offset 1/),EXPECTED_RETRY_FLAGS).returns(3)
121
+ @connection.expects(:select_value).with(regexp_matches(/where id >= 4 order by id limit 1 offset 1/),EXPECTED_RETRY_FLAGS).returns(5)
122
+ @connection.expects(:select_value).with(regexp_matches(/where id >= 6 order by id limit 1 offset 1/),EXPECTED_RETRY_FLAGS).returns(7)
123
+ @connection.expects(:select_value).with(regexp_matches(/where id >= 8 order by id limit 1 offset 1/),EXPECTED_RETRY_FLAGS).returns(9)
124
+ @connection.expects(:select_value).with(regexp_matches(/where id >= 10 order by id limit 1 offset 1/),EXPECTED_RETRY_FLAGS).returns(nil)
122
125
 
123
- @connection.expects(:update).with(regexp_matches(/between 2 and 3/),{}).returns(2)
124
- @connection.expects(:update).with(regexp_matches(/between 4 and 5/),{}).returns(2)
125
- @connection.expects(:update).with(regexp_matches(/between 6 and 7/),{}).returns(2)
126
- @connection.expects(:update).with(regexp_matches(/between 8 and 9/),{}).returns(2)
127
- @connection.expects(:update).with(regexp_matches(/between 10 and 10/),{}).returns(1)
126
+ @connection.expects(:update).with(regexp_matches(/between 2 and 3/),EXPECTED_RETRY_FLAGS).returns(2)
127
+ @connection.expects(:update).with(regexp_matches(/between 4 and 5/),EXPECTED_RETRY_FLAGS).returns(2)
128
+ @connection.expects(:update).with(regexp_matches(/between 6 and 7/),EXPECTED_RETRY_FLAGS).returns(2)
129
+ @connection.expects(:update).with(regexp_matches(/between 8 and 9/),EXPECTED_RETRY_FLAGS).returns(2)
130
+ @connection.expects(:update).with(regexp_matches(/between 10 and 10/),EXPECTED_RETRY_FLAGS).returns(1)
128
131
 
129
132
  @chunker.run
130
133
  end
@@ -138,9 +141,9 @@ describe Lhm::Chunker do
138
141
  2
139
142
  end
140
143
 
141
- @connection.expects(:select_value).with(regexp_matches(/where id >= 1 order by id limit 1 offset 1/),{}).returns(2)
142
- @connection.expects(:update).with(regexp_matches(/where \(foo.created_at > '2013-07-10' or foo.baz = 'quux'\) and `foo`/),{}).returns(1)
143
- @connection.expects(:execute).with(regexp_matches(/show warnings/),{}).returns([])
144
+ @connection.expects(:select_value).with(regexp_matches(/where id >= 1 order by id limit 1 offset 1/),EXPECTED_RETRY_FLAGS).returns(2)
145
+ @connection.expects(:update).with(regexp_matches(/where \(foo.created_at > '2013-07-10' or foo.baz = 'quux'\) and `foo`/),EXPECTED_RETRY_FLAGS).returns(1)
146
+ @connection.expects(:execute).with(regexp_matches(/show warnings/),EXPECTED_RETRY_FLAGS).returns([])
144
147
 
145
148
  def @migration.conditions
146
149
  "where foo.created_at > '2013-07-10' or foo.baz = 'quux'"
@@ -158,9 +161,9 @@ describe Lhm::Chunker do
158
161
  2
159
162
  end
160
163
 
161
- @connection.expects(:select_value).with(regexp_matches(/where id >= 1 order by id limit 1 offset 1/),{}).returns(2)
162
- @connection.expects(:update).with(regexp_matches(/inner join bar on foo.id = bar.foo_id and/),{}).returns(1)
163
- @connection.expects(:execute).with(regexp_matches(/show warnings/),{}).returns([])
164
+ @connection.expects(:select_value).with(regexp_matches(/where id >= 1 order by id limit 1 offset 1/),EXPECTED_RETRY_FLAGS).returns(2)
165
+ @connection.expects(:update).with(regexp_matches(/inner join bar on foo.id = bar.foo_id and/),EXPECTED_RETRY_FLAGS).returns(1)
166
+ @connection.expects(:execute).with(regexp_matches(/show warnings/),EXPECTED_RETRY_FLAGS).returns([])
164
167
 
165
168
  def @migration.conditions
166
169
  'inner join bar on foo.id = bar.foo_id'
@@ -1,4 +1,5 @@
1
1
  require 'lhm/connection'
2
+ require 'lhm/proxysql_helper'
2
3
 
3
4
  describe Lhm::Connection do
4
5
 
@@ -12,10 +13,11 @@ describe Lhm::Connection do
12
13
  it "Should find use calling file as prefix" do
13
14
  ar_connection = mock()
14
15
  ar_connection.stubs(:execute).raises(LOCK_WAIT).then.returns(true)
16
+ ar_connection.stubs(:active?).returns(true)
15
17
 
16
18
  connection = Lhm::Connection.new(connection: ar_connection)
17
19
 
18
- connection.execute("SHOW TABLES", { base_interval: 0 })
20
+ connection.execute("SHOW TABLES", should_retry: true, retry_options: { base_interval: 0 })
19
21
 
20
22
  log_messages = @logs.string.split("\n")
21
23
  assert_equal(1, log_messages.length)
@@ -27,10 +29,11 @@ describe Lhm::Connection do
27
29
  ar_connection.stubs(:execute).raises(LOCK_WAIT)
28
30
  .then.raises(LOCK_WAIT)
29
31
  .then.returns(true)
32
+ ar_connection.stubs(:active?).returns(true)
30
33
 
31
34
  connection = Lhm::Connection.new(connection: ar_connection)
32
35
 
33
- connection.execute("SHOW TABLES", { base_interval: 0, tries: 3 })
36
+ connection.execute("SHOW TABLES", should_retry: true, retry_options: { base_interval: 0, tries: 3 })
34
37
 
35
38
  log_messages = @logs.string.split("\n")
36
39
  assert_equal(2, log_messages.length)
@@ -41,10 +44,11 @@ describe Lhm::Connection do
41
44
  ar_connection.stubs(:update).raises(LOCK_WAIT)
42
45
  .then.raises(LOCK_WAIT)
43
46
  .then.returns(1)
47
+ ar_connection.stubs(:active?).returns(true)
44
48
 
45
49
  connection = Lhm::Connection.new(connection: ar_connection)
46
50
 
47
- val = connection.update("SHOW TABLES", { base_interval: 0, tries: 3 })
51
+ val = connection.update("SHOW TABLES", should_retry: true, retry_options:{ base_interval: 0, tries: 3 })
48
52
 
49
53
  log_messages = @logs.string.split("\n")
50
54
  assert_equal val, 1
@@ -56,13 +60,27 @@ describe Lhm::Connection do
56
60
  ar_connection.stubs(:select_value).raises(LOCK_WAIT)
57
61
  .then.raises(LOCK_WAIT)
58
62
  .then.returns("dummy")
63
+ ar_connection.stubs(:active?).returns(true)
59
64
 
60
65
  connection = Lhm::Connection.new(connection: ar_connection)
61
66
 
62
- val = connection.select_value("SHOW TABLES", { base_interval: 0, tries: 3 })
67
+ val = connection.select_value("SHOW TABLES", should_retry: true, retry_options: { base_interval: 0, tries: 3 })
63
68
 
64
69
  log_messages = @logs.string.split("\n")
65
70
  assert_equal val, "dummy"
66
71
  assert_equal(2, log_messages.length)
67
72
  end
73
+
74
+ it "Queries should be tagged with ProxySQL tag if requested" do
75
+ ar_connection = mock()
76
+ ar_connection.expects(:public_send).with(:select_value, "#{Lhm::ProxySQLHelper::ANNOTATION}SHOW TABLES").returns("dummy")
77
+ ar_connection.stubs(:execute).times(4).returns([["dummy"]])
78
+ ar_connection.stubs(:active?).returns(true)
79
+
80
+ connection = Lhm::Connection.new(connection: ar_connection, options: { reconnect_with_consistent_host: true })
81
+
82
+ val = connection.select_value("SHOW TABLES", should_retry: true, retry_options: { base_interval: 0, tries: 3 })
83
+
84
+ assert_equal val, "dummy"
85
+ end
68
86
  end
@@ -63,9 +63,13 @@ describe Lhm::Entangler do
63
63
  it 'should retry trigger creation when it hits a lock wait timeout' do
64
64
  tries = 1
65
65
  ar_connection = mock()
66
- ar_connection.expects(:execute).times(tries).raises(Mysql2::Error, 'Lock wait timeout exceeded; try restarting transaction')
66
+ ar_connection.stubs(:execute)
67
+ .returns([["dummy"]], [["dummy"]], [["dummy"]])
68
+ .then
69
+ .raises(Mysql2::Error, 'Lock wait timeout exceeded; try restarting transaction')
70
+ ar_connection.stubs(:active?).returns(true)
67
71
 
68
- connection = Lhm::Connection.new(connection: ar_connection)
72
+ connection = Lhm::Connection.new(connection: ar_connection, options: {reconnect_with_consistent_host: true})
69
73
 
70
74
  @entangler = Lhm::Entangler.new(@migration, connection, retriable: {base_interval: 0, tries: tries})
71
75
 
@@ -74,18 +78,28 @@ describe Lhm::Entangler do
74
78
 
75
79
  it 'should not retry trigger creation with other mysql errors' do
76
80
  ar_connection = mock()
77
- ar_connection.expects(:execute).once.raises(Mysql2::Error, 'The MySQL server is running with the --read-only option so it cannot execute this statement.')
78
- connection = Lhm::Connection.new(connection: ar_connection)
79
-
80
- @entangler = Lhm::Entangler.new(@migration, connection, retriable: {base_interval: 0})
81
+ ar_connection.stubs(:execute)
82
+ .returns([["dummy"]], [["dummy"]], [["dummy"]])
83
+ .then
84
+ .raises(Mysql2::Error, 'The MySQL server is running with the --read-only option so it cannot execute this statement.')
85
+ ar_connection.stubs(:active?).returns(true)
86
+ connection = Lhm::Connection.new(connection: ar_connection, options: {reconnect_with_consistent_host: true})
87
+
88
+ @entangler = Lhm::Entangler.new(@migration, connection, retriable: { base_interval: 0 })
81
89
  assert_raises(Mysql2::Error) { @entangler.before }
82
90
  end
83
91
 
84
92
  it 'should succesfully finish after retrying' do
85
93
  ar_connection = mock()
86
- ar_connection.stubs(:execute).raises(Mysql2::Error, 'Lock wait timeout exceeded; try restarting transaction').then.returns(true)
94
+ ar_connection.stubs(:execute)
95
+ .returns([["dummy"]], [["dummy"]], [["dummy"]])
96
+ .then
97
+ .raises(Mysql2::Error, 'Lock wait timeout exceeded; try restarting transaction')
98
+ .then
99
+ .returns([["dummy"]])
100
+ ar_connection.stubs(:active?).returns(true)
87
101
 
88
- connection = Lhm::Connection.new(connection: ar_connection)
102
+ connection = Lhm::Connection.new(connection: ar_connection, options: {reconnect_with_consistent_host: true})
89
103
 
90
104
  @entangler = Lhm::Entangler.new(@migration, connection, retriable: {base_interval: 0})
91
105
 
@@ -94,9 +108,25 @@ describe Lhm::Entangler do
94
108
 
95
109
  it 'should retry as many times as specified by configuration' do
96
110
  ar_connection = mock()
97
- ar_connection.expects(:execute).times(5).raises(Mysql2::Error, 'Lock wait timeout exceeded; try restarting transaction')
98
-
99
- connection = Lhm::Connection.new(connection: ar_connection)
111
+ ar_connection.stubs(:execute)
112
+ .returns([["dummy"]], [["dummy"]], [["dummy"]]) # initial
113
+ .then
114
+ .raises(Mysql2::Error, 'Lock wait timeout exceeded; try restarting transaction')
115
+ .then
116
+ .returns([["dummy"]]) # reconnect 1
117
+ .then
118
+ .raises(Mysql2::Error, 'Lock wait timeout exceeded; try restarting transaction')
119
+ .then
120
+ .returns([["dummy"]]) # reconnect 2
121
+ .then
122
+ .raises(Mysql2::Error, 'Lock wait timeout exceeded; try restarting transaction')
123
+ .then
124
+ .returns([["dummy"]]) # reconnect 3
125
+ .then
126
+ .raises(Mysql2::Error, 'Lock wait timeout exceeded; try restarting transaction') # final error
127
+ ar_connection.stubs(:active?).returns(true)
128
+
129
+ connection = Lhm::Connection.new(connection: ar_connection, options: {reconnect_with_consistent_host: true})
100
130
 
101
131
  @entangler = Lhm::Entangler.new(@migration, connection, retriable: {tries: 5, base_interval: 0})
102
132
 
@@ -39,7 +39,7 @@ describe Lhm::Throttler::Slave do
39
39
  @logs = StringIO.new
40
40
  Lhm.logger = Logger.new(@logs)
41
41
 
42
- @dummy_mysql_client_config = lambda { {'username' => 'user', 'password' => 'pw', 'database' => 'db'} }
42
+ @dummy_mysql_client_config = lambda { { 'username' => 'user', 'password' => 'pw', 'database' => 'db' } }
43
43
  end
44
44
 
45
45
  describe "#client" do
@@ -64,7 +64,7 @@ describe Lhm::Throttler::Slave do
64
64
 
65
65
  describe 'with proper config' do
66
66
  it "creates a new Mysql2::Client" do
67
- expected_config = {username: 'user', password: 'pw', database: 'db', host: 'slave'}
67
+ expected_config = { username: 'user', password: 'pw', database: 'db', host: 'slave' }
68
68
  Mysql2::Client.stubs(:new).with(expected_config).returns(mock())
69
69
 
70
70
  assert Lhm::Throttler::Slave.new('slave', @dummy_mysql_client_config).connection
@@ -73,8 +73,12 @@ describe Lhm::Throttler::Slave do
73
73
 
74
74
  describe 'with active record config' do
75
75
  it 'logs and creates client' do
76
- active_record_config = {username: 'user', password: 'pw', database: 'db'}
77
- ActiveRecord::Base.stubs(:connection_pool).returns(stub(spec: stub(config: active_record_config)))
76
+ active_record_config = { username: 'user', password: 'pw', database: 'db' }
77
+ if ActiveRecord::VERSION::MAJOR > 6 || ActiveRecord::VERSION::MAJOR == 6 && ActiveRecord::VERSION::MINOR >= 1
78
+ ActiveRecord::Base.stubs(:connection_pool).returns(stub(db_config: stub(configuration_hash: active_record_config)))
79
+ else
80
+ ActiveRecord::Base.stubs(:connection_pool).returns(stub(spec: stub(config: active_record_config)))
81
+ end
78
82
 
79
83
  Mysql2::Client.stubs(:new).returns(mock())
80
84
 
@@ -92,9 +96,9 @@ describe Lhm::Throttler::Slave do
92
96
  class Connection
93
97
  def self.query(query)
94
98
  if query == Lhm::Throttler::Slave::SQL_SELECT_MAX_SLAVE_LAG
95
- [{'Seconds_Behind_Master' => 20}]
99
+ [{ 'Seconds_Behind_Master' => 20 }]
96
100
  elsif query == Lhm::Throttler::Slave::SQL_SELECT_SLAVE_HOSTS
97
- [{'host' => '1.1.1.1:80'}]
101
+ [{ 'host' => '1.1.1.1:80' }]
98
102
  end
99
103
  end
100
104
  end
@@ -104,7 +108,7 @@ describe Lhm::Throttler::Slave do
104
108
 
105
109
  class StoppedConnection
106
110
  def self.query(query)
107
- [{'Seconds_Behind_Master' => nil}]
111
+ [{ 'Seconds_Behind_Master' => nil }]
108
112
  end
109
113
  end
110
114
 
@@ -286,7 +290,7 @@ describe Lhm::Throttler::SlaveLag do
286
290
  describe 'with the :check_only option' do
287
291
  describe 'with a callable argument' do
288
292
  before do
289
- check_only = lambda {{'host' => '1.1.1.3'}}
293
+ check_only = lambda { { 'host' => '1.1.1.3' } }
290
294
  @throttler = Lhm::Throttler::SlaveLag.new :check_only => check_only
291
295
  end
292
296
 
@@ -300,6 +304,7 @@ describe Lhm::Throttler::SlaveLag do
300
304
  describe 'with a non-callable argument' do
301
305
  before do
302
306
  @throttler = Lhm::Throttler::SlaveLag.new :check_only => 'I cannot be called'
307
+
303
308
  def @throttler.master_slave_hosts
304
309
  ['1.1.1.1', '1.1.1.4']
305
310
  end
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: lhm-shopify
3
3
  version: !ruby/object:Gem::Version
4
- version: 3.5.0
4
+ version: 3.5.1
5
5
  platform: ruby
6
6
  authors:
7
7
  - SoundCloud
@@ -12,7 +12,7 @@ authors:
12
12
  autorequire:
13
13
  bindir: bin
14
14
  cert_chain: []
15
- date: 2021-12-07 00:00:00.000000000 Z
15
+ date: 2021-12-06 00:00:00.000000000 Z
16
16
  dependencies:
17
17
  - !ruby/object:Gem::Dependency
18
18
  name: retriable
@@ -28,6 +28,20 @@ dependencies:
28
28
  - - ">="
29
29
  - !ruby/object:Gem::Version
30
30
  version: 3.0.0
31
+ - !ruby/object:Gem::Dependency
32
+ name: activerecord
33
+ requirement: !ruby/object:Gem::Requirement
34
+ requirements:
35
+ - - ">="
36
+ - !ruby/object:Gem::Version
37
+ version: '0'
38
+ type: :development
39
+ prerelease: false
40
+ version_requirements: !ruby/object:Gem::Requirement
41
+ requirements:
42
+ - - ">="
43
+ - !ruby/object:Gem::Version
44
+ version: '0'
31
45
  - !ruby/object:Gem::Dependency
32
46
  name: minitest
33
47
  requirement: !ruby/object:Gem::Requirement
@@ -57,7 +71,7 @@ dependencies:
57
71
  - !ruby/object:Gem::Version
58
72
  version: '0'
59
73
  - !ruby/object:Gem::Dependency
60
- name: rake
74
+ name: after_do
61
75
  requirement: !ruby/object:Gem::Requirement
62
76
  requirements:
63
77
  - - ">="
@@ -71,7 +85,7 @@ dependencies:
71
85
  - !ruby/object:Gem::Version
72
86
  version: '0'
73
87
  - !ruby/object:Gem::Dependency
74
- name: activerecord
88
+ name: rake
75
89
  requirement: !ruby/object:Gem::Requirement
76
90
  requirements:
77
91
  - - ">="
@@ -112,6 +126,48 @@ dependencies:
112
126
  - - ">="
113
127
  - !ruby/object:Gem::Version
114
128
  version: '0'
129
+ - !ruby/object:Gem::Dependency
130
+ name: toxiproxy
131
+ requirement: !ruby/object:Gem::Requirement
132
+ requirements:
133
+ - - ">="
134
+ - !ruby/object:Gem::Version
135
+ version: '0'
136
+ type: :development
137
+ prerelease: false
138
+ version_requirements: !ruby/object:Gem::Requirement
139
+ requirements:
140
+ - - ">="
141
+ - !ruby/object:Gem::Version
142
+ version: '0'
143
+ - !ruby/object:Gem::Dependency
144
+ name: appraisal
145
+ requirement: !ruby/object:Gem::Requirement
146
+ requirements:
147
+ - - ">="
148
+ - !ruby/object:Gem::Version
149
+ version: '0'
150
+ type: :development
151
+ prerelease: false
152
+ version_requirements: !ruby/object:Gem::Requirement
153
+ requirements:
154
+ - - ">="
155
+ - !ruby/object:Gem::Version
156
+ version: '0'
157
+ - !ruby/object:Gem::Dependency
158
+ name: byebug
159
+ requirement: !ruby/object:Gem::Requirement
160
+ requirements:
161
+ - - ">="
162
+ - !ruby/object:Gem::Version
163
+ version: '0'
164
+ type: :development
165
+ prerelease: false
166
+ version_requirements: !ruby/object:Gem::Requirement
167
+ requirements:
168
+ - - ">="
169
+ - !ruby/object:Gem::Version
170
+ version: '0'
115
171
  description: Migrate large tables without downtime by copying to a temporary table
116
172
  in chunks. The old table is not dropped. Instead, it is moved to timestamp_table_name
117
173
  for verification.
@@ -124,20 +180,23 @@ files:
124
180
  - ".gitignore"
125
181
  - ".rubocop.yml"
126
182
  - ".travis.yml"
183
+ - Appraisals
127
184
  - CHANGELOG.md
128
185
  - Gemfile
186
+ - Gemfile.lock
129
187
  - LICENSE
130
188
  - README.md
131
189
  - Rakefile
132
190
  - dev.yml
133
191
  - docker-compose.yml
134
- - gemfiles/ar-2.3_mysql.gemfile
135
- - gemfiles/ar-3.2_mysql.gemfile
136
- - gemfiles/ar-3.2_mysql2.gemfile
137
- - gemfiles/ar-4.0_mysql2.gemfile
138
- - gemfiles/ar-4.1_mysql2.gemfile
139
- - gemfiles/ar-4.2_mysql2.gemfile
140
- - gemfiles/ar-5.0_mysql2.gemfile
192
+ - gemfiles/activerecord_5.2.gemfile
193
+ - gemfiles/activerecord_5.2.gemfile.lock
194
+ - gemfiles/activerecord_6.0.gemfile
195
+ - gemfiles/activerecord_6.0.gemfile.lock
196
+ - gemfiles/activerecord_6.1.gemfile
197
+ - gemfiles/activerecord_6.1.gemfile.lock
198
+ - gemfiles/activerecord_7.0.0.alpha2.gemfile
199
+ - gemfiles/activerecord_7.0.0.alpha2.gemfile.lock
141
200
  - lhm.gemspec
142
201
  - lib/lhm-shopify.rb
143
202
  - lib/lhm.rb
@@ -155,6 +214,7 @@ files:
155
214
  - lib/lhm/migration.rb
156
215
  - lib/lhm/migrator.rb
157
216
  - lib/lhm/printer.rb
217
+ - lib/lhm/proxysql_helper.rb
158
218
  - lib/lhm/railtie.rb
159
219
  - lib/lhm/sql_helper.rb
160
220
  - lib/lhm/sql_retry.rb
@@ -199,9 +259,14 @@ files:
199
259
  - spec/integration/lhm_spec.rb
200
260
  - spec/integration/lock_wait_timeout_spec.rb
201
261
  - spec/integration/locked_switcher_spec.rb
262
+ - spec/integration/proxysql_spec.rb
263
+ - spec/integration/sql_retry/db_connection_helper.rb
202
264
  - spec/integration/sql_retry/lock_wait_spec.rb
203
265
  - spec/integration/sql_retry/lock_wait_timeout_test_helper.rb
266
+ - spec/integration/sql_retry/proxysql_helper.rb
267
+ - spec/integration/sql_retry/retry_with_proxysql_spec.rb
204
268
  - spec/integration/table_spec.rb
269
+ - spec/integration/toxiproxy_helper.rb
205
270
  - spec/test_helper.rb
206
271
  - spec/unit/atomic_switcher_spec.rb
207
272
  - spec/unit/chunk_finder_spec.rb
@@ -1,6 +0,0 @@
1
- source "https://rubygems.org"
2
-
3
- gem "mysql", "~> 2.8.1"
4
- gem "activerecord", "~> 2.3.18"
5
- gem "iconv", "~> 1.0.4"
6
- gemspec :path=>"../"
@@ -1,5 +0,0 @@
1
- source "https://rubygems.org"
2
-
3
- gem "mysql", "~> 2.8.1"
4
- gem "activerecord", "~> 3.2.2"
5
- gemspec :path=>"../"
@@ -1,5 +0,0 @@
1
- source "https://rubygems.org"
2
-
3
- gem "mysql2", "~> 0.3.11"
4
- gem "activerecord", "~> 3.2.2"
5
- gemspec :path=>"../"
@@ -1,5 +0,0 @@
1
- source "https://rubygems.org"
2
-
3
- gem "mysql2", "~> 0.3.17"
4
- gem "activerecord", "~> 4.0.13"
5
- gemspec :path=>"../"
@@ -1,5 +0,0 @@
1
- source "https://rubygems.org"
2
-
3
- gem "mysql2", "~> 0.3.17"
4
- gem "activerecord", "~> 4.1.9"
5
- gemspec :path=>"../"
@@ -1,5 +0,0 @@
1
- source "https://rubygems.org"
2
-
3
- gem "mysql2", "~> 0.3.17"
4
- gem "activerecord", "~> 4.2.0"
5
- gemspec :path=>"../"
@@ -1,5 +0,0 @@
1
- source "https://rubygems.org"
2
-
3
- gem "mysql2", "~> 0.4.5"
4
- gem "activerecord", "~> 5.0.2"
5
- gemspec :path=>"../"