lhm-shopify 3.5.0 → 3.5.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (59) hide show
  1. checksums.yaml +4 -4
  2. data/.github/workflows/test.yml +17 -4
  3. data/.gitignore +0 -2
  4. data/Appraisals +24 -0
  5. data/Gemfile.lock +66 -0
  6. data/README.md +42 -0
  7. data/Rakefile +1 -0
  8. data/dev.yml +18 -3
  9. data/docker-compose.yml +15 -3
  10. data/gemfiles/activerecord_5.2.gemfile +9 -0
  11. data/gemfiles/activerecord_5.2.gemfile.lock +65 -0
  12. data/gemfiles/activerecord_6.0.gemfile +7 -0
  13. data/gemfiles/activerecord_6.0.gemfile.lock +67 -0
  14. data/gemfiles/activerecord_6.1.gemfile +7 -0
  15. data/gemfiles/activerecord_6.1.gemfile.lock +66 -0
  16. data/gemfiles/activerecord_7.0.0.alpha2.gemfile +7 -0
  17. data/gemfiles/activerecord_7.0.0.alpha2.gemfile.lock +64 -0
  18. data/lhm.gemspec +7 -3
  19. data/lib/lhm/atomic_switcher.rb +1 -1
  20. data/lib/lhm/chunk_insert.rb +2 -1
  21. data/lib/lhm/chunker.rb +3 -3
  22. data/lib/lhm/cleanup/current.rb +1 -1
  23. data/lib/lhm/connection.rb +50 -11
  24. data/lib/lhm/entangler.rb +2 -2
  25. data/lib/lhm/invoker.rb +2 -2
  26. data/lib/lhm/proxysql_helper.rb +10 -0
  27. data/lib/lhm/sql_retry.rb +126 -8
  28. data/lib/lhm/throttler/slave_lag.rb +19 -2
  29. data/lib/lhm/version.rb +1 -1
  30. data/lib/lhm.rb +22 -8
  31. data/scripts/mysql/writer/create_users.sql +3 -0
  32. data/spec/integration/atomic_switcher_spec.rb +27 -10
  33. data/spec/integration/chunk_insert_spec.rb +2 -1
  34. data/spec/integration/chunker_spec.rb +1 -1
  35. data/spec/integration/database.yml +10 -0
  36. data/spec/integration/entangler_spec.rb +3 -1
  37. data/spec/integration/integration_helper.rb +23 -5
  38. data/spec/integration/lhm_spec.rb +75 -0
  39. data/spec/integration/proxysql_spec.rb +34 -0
  40. data/spec/integration/sql_retry/db_connection_helper.rb +52 -0
  41. data/spec/integration/sql_retry/lock_wait_spec.rb +8 -6
  42. data/spec/integration/sql_retry/lock_wait_timeout_test_helper.rb +19 -9
  43. data/spec/integration/sql_retry/proxysql_helper.rb +22 -0
  44. data/spec/integration/sql_retry/retry_with_proxysql_spec.rb +108 -0
  45. data/spec/integration/toxiproxy_helper.rb +40 -0
  46. data/spec/test_helper.rb +21 -0
  47. data/spec/unit/chunk_insert_spec.rb +7 -2
  48. data/spec/unit/chunker_spec.rb +45 -42
  49. data/spec/unit/connection_spec.rb +22 -4
  50. data/spec/unit/entangler_spec.rb +41 -11
  51. data/spec/unit/throttler/slave_lag_spec.rb +13 -8
  52. metadata +76 -11
  53. data/gemfiles/ar-2.3_mysql.gemfile +0 -6
  54. data/gemfiles/ar-3.2_mysql.gemfile +0 -5
  55. data/gemfiles/ar-3.2_mysql2.gemfile +0 -5
  56. data/gemfiles/ar-4.0_mysql2.gemfile +0 -5
  57. data/gemfiles/ar-4.1_mysql2.gemfile +0 -5
  58. data/gemfiles/ar-4.2_mysql2.gemfile +0 -5
  59. data/gemfiles/ar-5.0_mysql2.gemfile +0 -5
@@ -12,11 +12,14 @@ require 'lhm/connection'
12
12
  describe Lhm::Chunker do
13
13
  include UnitHelper
14
14
 
15
+ EXPECTED_RETRY_FLAGS = {:should_retry => true, :retry_options => {}}
16
+
15
17
  before(:each) do
16
18
  @origin = Lhm::Table.new('foo')
17
19
  @destination = Lhm::Table.new('bar')
18
20
  @migration = Lhm::Migration.new(@origin, @destination)
19
21
  @connection = mock()
22
+ @connection.stubs(:execute).returns([["dummy"]])
20
23
  # This is a poor man's stub
21
24
  @throttler = Object.new
22
25
  def @throttler.run
@@ -38,11 +41,11 @@ describe Lhm::Chunker do
38
41
  5
39
42
  end
40
43
 
41
- @connection.expects(:select_value).with(regexp_matches(/where id >= 1 order by id limit 1 offset 4/),{}).returns(7)
42
- @connection.expects(:select_value).with(regexp_matches(/where id >= 8 order by id limit 1 offset 4/),{}).returns(21)
43
- @connection.expects(:update).with(regexp_matches(/between 1 and 7/),{}).returns(2)
44
- @connection.expects(:update).with(regexp_matches(/between 8 and 10/),{}).returns(2)
45
- @connection.expects(:execute).twice.with(regexp_matches(/show warnings/),{}).returns([])
44
+ @connection.expects(:select_value).with(regexp_matches(/where id >= 1 order by id limit 1 offset 4/),EXPECTED_RETRY_FLAGS).returns(7)
45
+ @connection.expects(:select_value).with(regexp_matches(/where id >= 8 order by id limit 1 offset 4/),EXPECTED_RETRY_FLAGS).returns(21)
46
+ @connection.expects(:update).with(regexp_matches(/between 1 and 7/),EXPECTED_RETRY_FLAGS).returns(2)
47
+ @connection.expects(:update).with(regexp_matches(/between 8 and 10/),EXPECTED_RETRY_FLAGS).returns(2)
48
+ @connection.expects(:execute).twice.with(regexp_matches(/show warnings/),EXPECTED_RETRY_FLAGS).returns([])
46
49
 
47
50
  @chunker.run
48
51
  end
@@ -53,17 +56,17 @@ describe Lhm::Chunker do
53
56
  2
54
57
  end
55
58
 
56
- @connection.expects(:select_value).with(regexp_matches(/where id >= 1 order by id limit 1 offset 1/),{}).returns(2)
57
- @connection.expects(:select_value).with(regexp_matches(/where id >= 3 order by id limit 1 offset 1/),{}).returns(4)
58
- @connection.expects(:select_value).with(regexp_matches(/where id >= 5 order by id limit 1 offset 1/),{}).returns(6)
59
- @connection.expects(:select_value).with(regexp_matches(/where id >= 7 order by id limit 1 offset 1/),{}).returns(8)
60
- @connection.expects(:select_value).with(regexp_matches(/where id >= 9 order by id limit 1 offset 1/),{}).returns(10)
59
+ @connection.expects(:select_value).with(regexp_matches(/where id >= 1 order by id limit 1 offset 1/),EXPECTED_RETRY_FLAGS).returns(2)
60
+ @connection.expects(:select_value).with(regexp_matches(/where id >= 3 order by id limit 1 offset 1/),EXPECTED_RETRY_FLAGS).returns(4)
61
+ @connection.expects(:select_value).with(regexp_matches(/where id >= 5 order by id limit 1 offset 1/),EXPECTED_RETRY_FLAGS).returns(6)
62
+ @connection.expects(:select_value).with(regexp_matches(/where id >= 7 order by id limit 1 offset 1/),EXPECTED_RETRY_FLAGS).returns(8)
63
+ @connection.expects(:select_value).with(regexp_matches(/where id >= 9 order by id limit 1 offset 1/),EXPECTED_RETRY_FLAGS).returns(10)
61
64
 
62
- @connection.expects(:update).with(regexp_matches(/between 1 and 2/),{}).returns(2)
63
- @connection.expects(:update).with(regexp_matches(/between 3 and 4/),{}).returns(2)
64
- @connection.expects(:update).with(regexp_matches(/between 5 and 6/),{}).returns(2)
65
- @connection.expects(:update).with(regexp_matches(/between 7 and 8/),{}).returns(2)
66
- @connection.expects(:update).with(regexp_matches(/between 9 and 10/),{}).returns(2)
65
+ @connection.expects(:update).with(regexp_matches(/between 1 and 2/),EXPECTED_RETRY_FLAGS).returns(2)
66
+ @connection.expects(:update).with(regexp_matches(/between 3 and 4/),EXPECTED_RETRY_FLAGS).returns(2)
67
+ @connection.expects(:update).with(regexp_matches(/between 5 and 6/),EXPECTED_RETRY_FLAGS).returns(2)
68
+ @connection.expects(:update).with(regexp_matches(/between 7 and 8/),EXPECTED_RETRY_FLAGS).returns(2)
69
+ @connection.expects(:update).with(regexp_matches(/between 9 and 10/),EXPECTED_RETRY_FLAGS).returns(2)
67
70
 
68
71
  @chunker.run
69
72
  end
@@ -80,17 +83,17 @@ describe Lhm::Chunker do
80
83
  end
81
84
  end
82
85
 
83
- @connection.expects(:select_value).with(regexp_matches(/where id >= 1 order by id limit 1 offset 1/),{}).returns(2)
84
- @connection.expects(:select_value).with(regexp_matches(/where id >= 3 order by id limit 1 offset 2/),{}).returns(5)
85
- @connection.expects(:select_value).with(regexp_matches(/where id >= 6 order by id limit 1 offset 2/),{}).returns(8)
86
- @connection.expects(:select_value).with(regexp_matches(/where id >= 9 order by id limit 1 offset 2/),{}).returns(nil)
86
+ @connection.expects(:select_value).with(regexp_matches(/where id >= 1 order by id limit 1 offset 1/),EXPECTED_RETRY_FLAGS).returns(2)
87
+ @connection.expects(:select_value).with(regexp_matches(/where id >= 3 order by id limit 1 offset 2/),EXPECTED_RETRY_FLAGS).returns(5)
88
+ @connection.expects(:select_value).with(regexp_matches(/where id >= 6 order by id limit 1 offset 2/),EXPECTED_RETRY_FLAGS).returns(8)
89
+ @connection.expects(:select_value).with(regexp_matches(/where id >= 9 order by id limit 1 offset 2/),EXPECTED_RETRY_FLAGS).returns(nil)
87
90
 
88
- @connection.expects(:update).with(regexp_matches(/between 1 and 2/),{}).returns(2)
89
- @connection.expects(:update).with(regexp_matches(/between 3 and 5/),{}).returns(2)
90
- @connection.expects(:update).with(regexp_matches(/between 6 and 8/),{}).returns(2)
91
- @connection.expects(:update).with(regexp_matches(/between 9 and 10/),{}).returns(2)
91
+ @connection.expects(:update).with(regexp_matches(/between 1 and 2/),EXPECTED_RETRY_FLAGS).returns(2)
92
+ @connection.expects(:update).with(regexp_matches(/between 3 and 5/),EXPECTED_RETRY_FLAGS).returns(2)
93
+ @connection.expects(:update).with(regexp_matches(/between 6 and 8/),EXPECTED_RETRY_FLAGS).returns(2)
94
+ @connection.expects(:update).with(regexp_matches(/between 9 and 10/),EXPECTED_RETRY_FLAGS).returns(2)
92
95
 
93
- @connection.expects(:execute).twice.with(regexp_matches(/show warnings/),{}).returns([])
96
+ @connection.expects(:execute).twice.with(regexp_matches(/show warnings/),EXPECTED_RETRY_FLAGS).returns([])
94
97
 
95
98
  @chunker.run
96
99
  end
@@ -100,8 +103,8 @@ describe Lhm::Chunker do
100
103
  :start => 1,
101
104
  :limit => 1)
102
105
 
103
- @connection.expects(:select_value).with(regexp_matches(/where id >= 1 order by id limit 1 offset 0/),{}).returns(nil)
104
- @connection.expects(:update).with(regexp_matches(/between 1 and 1/),{}).returns(1)
106
+ @connection.expects(:select_value).with(regexp_matches(/where id >= 1 order by id limit 1 offset 0/),EXPECTED_RETRY_FLAGS).returns(nil)
107
+ @connection.expects(:update).with(regexp_matches(/between 1 and 1/),EXPECTED_RETRY_FLAGS).returns(1)
105
108
 
106
109
  @chunker.run
107
110
  end
@@ -114,17 +117,17 @@ describe Lhm::Chunker do
114
117
  2
115
118
  end
116
119
 
117
- @connection.expects(:select_value).with(regexp_matches(/where id >= 2 order by id limit 1 offset 1/),{}).returns(3)
118
- @connection.expects(:select_value).with(regexp_matches(/where id >= 4 order by id limit 1 offset 1/),{}).returns(5)
119
- @connection.expects(:select_value).with(regexp_matches(/where id >= 6 order by id limit 1 offset 1/),{}).returns(7)
120
- @connection.expects(:select_value).with(regexp_matches(/where id >= 8 order by id limit 1 offset 1/),{}).returns(9)
121
- @connection.expects(:select_value).with(regexp_matches(/where id >= 10 order by id limit 1 offset 1/),{}).returns(nil)
120
+ @connection.expects(:select_value).with(regexp_matches(/where id >= 2 order by id limit 1 offset 1/),EXPECTED_RETRY_FLAGS).returns(3)
121
+ @connection.expects(:select_value).with(regexp_matches(/where id >= 4 order by id limit 1 offset 1/),EXPECTED_RETRY_FLAGS).returns(5)
122
+ @connection.expects(:select_value).with(regexp_matches(/where id >= 6 order by id limit 1 offset 1/),EXPECTED_RETRY_FLAGS).returns(7)
123
+ @connection.expects(:select_value).with(regexp_matches(/where id >= 8 order by id limit 1 offset 1/),EXPECTED_RETRY_FLAGS).returns(9)
124
+ @connection.expects(:select_value).with(regexp_matches(/where id >= 10 order by id limit 1 offset 1/),EXPECTED_RETRY_FLAGS).returns(nil)
122
125
 
123
- @connection.expects(:update).with(regexp_matches(/between 2 and 3/),{}).returns(2)
124
- @connection.expects(:update).with(regexp_matches(/between 4 and 5/),{}).returns(2)
125
- @connection.expects(:update).with(regexp_matches(/between 6 and 7/),{}).returns(2)
126
- @connection.expects(:update).with(regexp_matches(/between 8 and 9/),{}).returns(2)
127
- @connection.expects(:update).with(regexp_matches(/between 10 and 10/),{}).returns(1)
126
+ @connection.expects(:update).with(regexp_matches(/between 2 and 3/),EXPECTED_RETRY_FLAGS).returns(2)
127
+ @connection.expects(:update).with(regexp_matches(/between 4 and 5/),EXPECTED_RETRY_FLAGS).returns(2)
128
+ @connection.expects(:update).with(regexp_matches(/between 6 and 7/),EXPECTED_RETRY_FLAGS).returns(2)
129
+ @connection.expects(:update).with(regexp_matches(/between 8 and 9/),EXPECTED_RETRY_FLAGS).returns(2)
130
+ @connection.expects(:update).with(regexp_matches(/between 10 and 10/),EXPECTED_RETRY_FLAGS).returns(1)
128
131
 
129
132
  @chunker.run
130
133
  end
@@ -138,9 +141,9 @@ describe Lhm::Chunker do
138
141
  2
139
142
  end
140
143
 
141
- @connection.expects(:select_value).with(regexp_matches(/where id >= 1 order by id limit 1 offset 1/),{}).returns(2)
142
- @connection.expects(:update).with(regexp_matches(/where \(foo.created_at > '2013-07-10' or foo.baz = 'quux'\) and `foo`/),{}).returns(1)
143
- @connection.expects(:execute).with(regexp_matches(/show warnings/),{}).returns([])
144
+ @connection.expects(:select_value).with(regexp_matches(/where id >= 1 order by id limit 1 offset 1/),EXPECTED_RETRY_FLAGS).returns(2)
145
+ @connection.expects(:update).with(regexp_matches(/where \(foo.created_at > '2013-07-10' or foo.baz = 'quux'\) and `foo`/),EXPECTED_RETRY_FLAGS).returns(1)
146
+ @connection.expects(:execute).with(regexp_matches(/show warnings/),EXPECTED_RETRY_FLAGS).returns([])
144
147
 
145
148
  def @migration.conditions
146
149
  "where foo.created_at > '2013-07-10' or foo.baz = 'quux'"
@@ -158,9 +161,9 @@ describe Lhm::Chunker do
158
161
  2
159
162
  end
160
163
 
161
- @connection.expects(:select_value).with(regexp_matches(/where id >= 1 order by id limit 1 offset 1/),{}).returns(2)
162
- @connection.expects(:update).with(regexp_matches(/inner join bar on foo.id = bar.foo_id and/),{}).returns(1)
163
- @connection.expects(:execute).with(regexp_matches(/show warnings/),{}).returns([])
164
+ @connection.expects(:select_value).with(regexp_matches(/where id >= 1 order by id limit 1 offset 1/),EXPECTED_RETRY_FLAGS).returns(2)
165
+ @connection.expects(:update).with(regexp_matches(/inner join bar on foo.id = bar.foo_id and/),EXPECTED_RETRY_FLAGS).returns(1)
166
+ @connection.expects(:execute).with(regexp_matches(/show warnings/),EXPECTED_RETRY_FLAGS).returns([])
164
167
 
165
168
  def @migration.conditions
166
169
  'inner join bar on foo.id = bar.foo_id'
@@ -1,4 +1,5 @@
1
1
  require 'lhm/connection'
2
+ require 'lhm/proxysql_helper'
2
3
 
3
4
  describe Lhm::Connection do
4
5
 
@@ -12,10 +13,11 @@ describe Lhm::Connection do
12
13
  it "Should find use calling file as prefix" do
13
14
  ar_connection = mock()
14
15
  ar_connection.stubs(:execute).raises(LOCK_WAIT).then.returns(true)
16
+ ar_connection.stubs(:active?).returns(true)
15
17
 
16
18
  connection = Lhm::Connection.new(connection: ar_connection)
17
19
 
18
- connection.execute("SHOW TABLES", { base_interval: 0 })
20
+ connection.execute("SHOW TABLES", should_retry: true, retry_options: { base_interval: 0 })
19
21
 
20
22
  log_messages = @logs.string.split("\n")
21
23
  assert_equal(1, log_messages.length)
@@ -27,10 +29,11 @@ describe Lhm::Connection do
27
29
  ar_connection.stubs(:execute).raises(LOCK_WAIT)
28
30
  .then.raises(LOCK_WAIT)
29
31
  .then.returns(true)
32
+ ar_connection.stubs(:active?).returns(true)
30
33
 
31
34
  connection = Lhm::Connection.new(connection: ar_connection)
32
35
 
33
- connection.execute("SHOW TABLES", { base_interval: 0, tries: 3 })
36
+ connection.execute("SHOW TABLES", should_retry: true, retry_options: { base_interval: 0, tries: 3 })
34
37
 
35
38
  log_messages = @logs.string.split("\n")
36
39
  assert_equal(2, log_messages.length)
@@ -41,10 +44,11 @@ describe Lhm::Connection do
41
44
  ar_connection.stubs(:update).raises(LOCK_WAIT)
42
45
  .then.raises(LOCK_WAIT)
43
46
  .then.returns(1)
47
+ ar_connection.stubs(:active?).returns(true)
44
48
 
45
49
  connection = Lhm::Connection.new(connection: ar_connection)
46
50
 
47
- val = connection.update("SHOW TABLES", { base_interval: 0, tries: 3 })
51
+ val = connection.update("SHOW TABLES", should_retry: true, retry_options:{ base_interval: 0, tries: 3 })
48
52
 
49
53
  log_messages = @logs.string.split("\n")
50
54
  assert_equal val, 1
@@ -56,13 +60,27 @@ describe Lhm::Connection do
56
60
  ar_connection.stubs(:select_value).raises(LOCK_WAIT)
57
61
  .then.raises(LOCK_WAIT)
58
62
  .then.returns("dummy")
63
+ ar_connection.stubs(:active?).returns(true)
59
64
 
60
65
  connection = Lhm::Connection.new(connection: ar_connection)
61
66
 
62
- val = connection.select_value("SHOW TABLES", { base_interval: 0, tries: 3 })
67
+ val = connection.select_value("SHOW TABLES", should_retry: true, retry_options: { base_interval: 0, tries: 3 })
63
68
 
64
69
  log_messages = @logs.string.split("\n")
65
70
  assert_equal val, "dummy"
66
71
  assert_equal(2, log_messages.length)
67
72
  end
73
+
74
+ it "Queries should be tagged with ProxySQL tag if requested" do
75
+ ar_connection = mock()
76
+ ar_connection.expects(:public_send).with(:select_value, "#{Lhm::ProxySQLHelper::ANNOTATION}SHOW TABLES").returns("dummy")
77
+ ar_connection.stubs(:execute).times(4).returns([["dummy"]])
78
+ ar_connection.stubs(:active?).returns(true)
79
+
80
+ connection = Lhm::Connection.new(connection: ar_connection, options: { reconnect_with_consistent_host: true })
81
+
82
+ val = connection.select_value("SHOW TABLES", should_retry: true, retry_options: { base_interval: 0, tries: 3 })
83
+
84
+ assert_equal val, "dummy"
85
+ end
68
86
  end
@@ -63,9 +63,13 @@ describe Lhm::Entangler do
63
63
  it 'should retry trigger creation when it hits a lock wait timeout' do
64
64
  tries = 1
65
65
  ar_connection = mock()
66
- ar_connection.expects(:execute).times(tries).raises(Mysql2::Error, 'Lock wait timeout exceeded; try restarting transaction')
66
+ ar_connection.stubs(:execute)
67
+ .returns([["dummy"]], [["dummy"]], [["dummy"]])
68
+ .then
69
+ .raises(Mysql2::Error, 'Lock wait timeout exceeded; try restarting transaction')
70
+ ar_connection.stubs(:active?).returns(true)
67
71
 
68
- connection = Lhm::Connection.new(connection: ar_connection)
72
+ connection = Lhm::Connection.new(connection: ar_connection, options: {reconnect_with_consistent_host: true})
69
73
 
70
74
  @entangler = Lhm::Entangler.new(@migration, connection, retriable: {base_interval: 0, tries: tries})
71
75
 
@@ -74,18 +78,28 @@ describe Lhm::Entangler do
74
78
 
75
79
  it 'should not retry trigger creation with other mysql errors' do
76
80
  ar_connection = mock()
77
- ar_connection.expects(:execute).once.raises(Mysql2::Error, 'The MySQL server is running with the --read-only option so it cannot execute this statement.')
78
- connection = Lhm::Connection.new(connection: ar_connection)
79
-
80
- @entangler = Lhm::Entangler.new(@migration, connection, retriable: {base_interval: 0})
81
+ ar_connection.stubs(:execute)
82
+ .returns([["dummy"]], [["dummy"]], [["dummy"]])
83
+ .then
84
+ .raises(Mysql2::Error, 'The MySQL server is running with the --read-only option so it cannot execute this statement.')
85
+ ar_connection.stubs(:active?).returns(true)
86
+ connection = Lhm::Connection.new(connection: ar_connection, options: {reconnect_with_consistent_host: true})
87
+
88
+ @entangler = Lhm::Entangler.new(@migration, connection, retriable: { base_interval: 0 })
81
89
  assert_raises(Mysql2::Error) { @entangler.before }
82
90
  end
83
91
 
84
92
  it 'should succesfully finish after retrying' do
85
93
  ar_connection = mock()
86
- ar_connection.stubs(:execute).raises(Mysql2::Error, 'Lock wait timeout exceeded; try restarting transaction').then.returns(true)
94
+ ar_connection.stubs(:execute)
95
+ .returns([["dummy"]], [["dummy"]], [["dummy"]])
96
+ .then
97
+ .raises(Mysql2::Error, 'Lock wait timeout exceeded; try restarting transaction')
98
+ .then
99
+ .returns([["dummy"]])
100
+ ar_connection.stubs(:active?).returns(true)
87
101
 
88
- connection = Lhm::Connection.new(connection: ar_connection)
102
+ connection = Lhm::Connection.new(connection: ar_connection, options: {reconnect_with_consistent_host: true})
89
103
 
90
104
  @entangler = Lhm::Entangler.new(@migration, connection, retriable: {base_interval: 0})
91
105
 
@@ -94,9 +108,25 @@ describe Lhm::Entangler do
94
108
 
95
109
  it 'should retry as many times as specified by configuration' do
96
110
  ar_connection = mock()
97
- ar_connection.expects(:execute).times(5).raises(Mysql2::Error, 'Lock wait timeout exceeded; try restarting transaction')
98
-
99
- connection = Lhm::Connection.new(connection: ar_connection)
111
+ ar_connection.stubs(:execute)
112
+ .returns([["dummy"]], [["dummy"]], [["dummy"]]) # initial
113
+ .then
114
+ .raises(Mysql2::Error, 'Lock wait timeout exceeded; try restarting transaction')
115
+ .then
116
+ .returns([["dummy"]]) # reconnect 1
117
+ .then
118
+ .raises(Mysql2::Error, 'Lock wait timeout exceeded; try restarting transaction')
119
+ .then
120
+ .returns([["dummy"]]) # reconnect 2
121
+ .then
122
+ .raises(Mysql2::Error, 'Lock wait timeout exceeded; try restarting transaction')
123
+ .then
124
+ .returns([["dummy"]]) # reconnect 3
125
+ .then
126
+ .raises(Mysql2::Error, 'Lock wait timeout exceeded; try restarting transaction') # final error
127
+ ar_connection.stubs(:active?).returns(true)
128
+
129
+ connection = Lhm::Connection.new(connection: ar_connection, options: {reconnect_with_consistent_host: true})
100
130
 
101
131
  @entangler = Lhm::Entangler.new(@migration, connection, retriable: {tries: 5, base_interval: 0})
102
132
 
@@ -39,7 +39,7 @@ describe Lhm::Throttler::Slave do
39
39
  @logs = StringIO.new
40
40
  Lhm.logger = Logger.new(@logs)
41
41
 
42
- @dummy_mysql_client_config = lambda { {'username' => 'user', 'password' => 'pw', 'database' => 'db'} }
42
+ @dummy_mysql_client_config = lambda { { 'username' => 'user', 'password' => 'pw', 'database' => 'db' } }
43
43
  end
44
44
 
45
45
  describe "#client" do
@@ -64,7 +64,7 @@ describe Lhm::Throttler::Slave do
64
64
 
65
65
  describe 'with proper config' do
66
66
  it "creates a new Mysql2::Client" do
67
- expected_config = {username: 'user', password: 'pw', database: 'db', host: 'slave'}
67
+ expected_config = { username: 'user', password: 'pw', database: 'db', host: 'slave' }
68
68
  Mysql2::Client.stubs(:new).with(expected_config).returns(mock())
69
69
 
70
70
  assert Lhm::Throttler::Slave.new('slave', @dummy_mysql_client_config).connection
@@ -73,8 +73,12 @@ describe Lhm::Throttler::Slave do
73
73
 
74
74
  describe 'with active record config' do
75
75
  it 'logs and creates client' do
76
- active_record_config = {username: 'user', password: 'pw', database: 'db'}
77
- ActiveRecord::Base.stubs(:connection_pool).returns(stub(spec: stub(config: active_record_config)))
76
+ active_record_config = { username: 'user', password: 'pw', database: 'db' }
77
+ if ActiveRecord::VERSION::MAJOR > 6 || ActiveRecord::VERSION::MAJOR == 6 && ActiveRecord::VERSION::MINOR >= 1
78
+ ActiveRecord::Base.stubs(:connection_pool).returns(stub(db_config: stub(configuration_hash: active_record_config)))
79
+ else
80
+ ActiveRecord::Base.stubs(:connection_pool).returns(stub(spec: stub(config: active_record_config)))
81
+ end
78
82
 
79
83
  Mysql2::Client.stubs(:new).returns(mock())
80
84
 
@@ -92,9 +96,9 @@ describe Lhm::Throttler::Slave do
92
96
  class Connection
93
97
  def self.query(query)
94
98
  if query == Lhm::Throttler::Slave::SQL_SELECT_MAX_SLAVE_LAG
95
- [{'Seconds_Behind_Master' => 20}]
99
+ [{ 'Seconds_Behind_Master' => 20 }]
96
100
  elsif query == Lhm::Throttler::Slave::SQL_SELECT_SLAVE_HOSTS
97
- [{'host' => '1.1.1.1:80'}]
101
+ [{ 'host' => '1.1.1.1:80' }]
98
102
  end
99
103
  end
100
104
  end
@@ -104,7 +108,7 @@ describe Lhm::Throttler::Slave do
104
108
 
105
109
  class StoppedConnection
106
110
  def self.query(query)
107
- [{'Seconds_Behind_Master' => nil}]
111
+ [{ 'Seconds_Behind_Master' => nil }]
108
112
  end
109
113
  end
110
114
 
@@ -286,7 +290,7 @@ describe Lhm::Throttler::SlaveLag do
286
290
  describe 'with the :check_only option' do
287
291
  describe 'with a callable argument' do
288
292
  before do
289
- check_only = lambda {{'host' => '1.1.1.3'}}
293
+ check_only = lambda { { 'host' => '1.1.1.3' } }
290
294
  @throttler = Lhm::Throttler::SlaveLag.new :check_only => check_only
291
295
  end
292
296
 
@@ -300,6 +304,7 @@ describe Lhm::Throttler::SlaveLag do
300
304
  describe 'with a non-callable argument' do
301
305
  before do
302
306
  @throttler = Lhm::Throttler::SlaveLag.new :check_only => 'I cannot be called'
307
+
303
308
  def @throttler.master_slave_hosts
304
309
  ['1.1.1.1', '1.1.1.4']
305
310
  end
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: lhm-shopify
3
3
  version: !ruby/object:Gem::Version
4
- version: 3.5.0
4
+ version: 3.5.1
5
5
  platform: ruby
6
6
  authors:
7
7
  - SoundCloud
@@ -12,7 +12,7 @@ authors:
12
12
  autorequire:
13
13
  bindir: bin
14
14
  cert_chain: []
15
- date: 2021-12-07 00:00:00.000000000 Z
15
+ date: 2021-12-06 00:00:00.000000000 Z
16
16
  dependencies:
17
17
  - !ruby/object:Gem::Dependency
18
18
  name: retriable
@@ -28,6 +28,20 @@ dependencies:
28
28
  - - ">="
29
29
  - !ruby/object:Gem::Version
30
30
  version: 3.0.0
31
+ - !ruby/object:Gem::Dependency
32
+ name: activerecord
33
+ requirement: !ruby/object:Gem::Requirement
34
+ requirements:
35
+ - - ">="
36
+ - !ruby/object:Gem::Version
37
+ version: '0'
38
+ type: :development
39
+ prerelease: false
40
+ version_requirements: !ruby/object:Gem::Requirement
41
+ requirements:
42
+ - - ">="
43
+ - !ruby/object:Gem::Version
44
+ version: '0'
31
45
  - !ruby/object:Gem::Dependency
32
46
  name: minitest
33
47
  requirement: !ruby/object:Gem::Requirement
@@ -57,7 +71,7 @@ dependencies:
57
71
  - !ruby/object:Gem::Version
58
72
  version: '0'
59
73
  - !ruby/object:Gem::Dependency
60
- name: rake
74
+ name: after_do
61
75
  requirement: !ruby/object:Gem::Requirement
62
76
  requirements:
63
77
  - - ">="
@@ -71,7 +85,7 @@ dependencies:
71
85
  - !ruby/object:Gem::Version
72
86
  version: '0'
73
87
  - !ruby/object:Gem::Dependency
74
- name: activerecord
88
+ name: rake
75
89
  requirement: !ruby/object:Gem::Requirement
76
90
  requirements:
77
91
  - - ">="
@@ -112,6 +126,48 @@ dependencies:
112
126
  - - ">="
113
127
  - !ruby/object:Gem::Version
114
128
  version: '0'
129
+ - !ruby/object:Gem::Dependency
130
+ name: toxiproxy
131
+ requirement: !ruby/object:Gem::Requirement
132
+ requirements:
133
+ - - ">="
134
+ - !ruby/object:Gem::Version
135
+ version: '0'
136
+ type: :development
137
+ prerelease: false
138
+ version_requirements: !ruby/object:Gem::Requirement
139
+ requirements:
140
+ - - ">="
141
+ - !ruby/object:Gem::Version
142
+ version: '0'
143
+ - !ruby/object:Gem::Dependency
144
+ name: appraisal
145
+ requirement: !ruby/object:Gem::Requirement
146
+ requirements:
147
+ - - ">="
148
+ - !ruby/object:Gem::Version
149
+ version: '0'
150
+ type: :development
151
+ prerelease: false
152
+ version_requirements: !ruby/object:Gem::Requirement
153
+ requirements:
154
+ - - ">="
155
+ - !ruby/object:Gem::Version
156
+ version: '0'
157
+ - !ruby/object:Gem::Dependency
158
+ name: byebug
159
+ requirement: !ruby/object:Gem::Requirement
160
+ requirements:
161
+ - - ">="
162
+ - !ruby/object:Gem::Version
163
+ version: '0'
164
+ type: :development
165
+ prerelease: false
166
+ version_requirements: !ruby/object:Gem::Requirement
167
+ requirements:
168
+ - - ">="
169
+ - !ruby/object:Gem::Version
170
+ version: '0'
115
171
  description: Migrate large tables without downtime by copying to a temporary table
116
172
  in chunks. The old table is not dropped. Instead, it is moved to timestamp_table_name
117
173
  for verification.
@@ -124,20 +180,23 @@ files:
124
180
  - ".gitignore"
125
181
  - ".rubocop.yml"
126
182
  - ".travis.yml"
183
+ - Appraisals
127
184
  - CHANGELOG.md
128
185
  - Gemfile
186
+ - Gemfile.lock
129
187
  - LICENSE
130
188
  - README.md
131
189
  - Rakefile
132
190
  - dev.yml
133
191
  - docker-compose.yml
134
- - gemfiles/ar-2.3_mysql.gemfile
135
- - gemfiles/ar-3.2_mysql.gemfile
136
- - gemfiles/ar-3.2_mysql2.gemfile
137
- - gemfiles/ar-4.0_mysql2.gemfile
138
- - gemfiles/ar-4.1_mysql2.gemfile
139
- - gemfiles/ar-4.2_mysql2.gemfile
140
- - gemfiles/ar-5.0_mysql2.gemfile
192
+ - gemfiles/activerecord_5.2.gemfile
193
+ - gemfiles/activerecord_5.2.gemfile.lock
194
+ - gemfiles/activerecord_6.0.gemfile
195
+ - gemfiles/activerecord_6.0.gemfile.lock
196
+ - gemfiles/activerecord_6.1.gemfile
197
+ - gemfiles/activerecord_6.1.gemfile.lock
198
+ - gemfiles/activerecord_7.0.0.alpha2.gemfile
199
+ - gemfiles/activerecord_7.0.0.alpha2.gemfile.lock
141
200
  - lhm.gemspec
142
201
  - lib/lhm-shopify.rb
143
202
  - lib/lhm.rb
@@ -155,6 +214,7 @@ files:
155
214
  - lib/lhm/migration.rb
156
215
  - lib/lhm/migrator.rb
157
216
  - lib/lhm/printer.rb
217
+ - lib/lhm/proxysql_helper.rb
158
218
  - lib/lhm/railtie.rb
159
219
  - lib/lhm/sql_helper.rb
160
220
  - lib/lhm/sql_retry.rb
@@ -199,9 +259,14 @@ files:
199
259
  - spec/integration/lhm_spec.rb
200
260
  - spec/integration/lock_wait_timeout_spec.rb
201
261
  - spec/integration/locked_switcher_spec.rb
262
+ - spec/integration/proxysql_spec.rb
263
+ - spec/integration/sql_retry/db_connection_helper.rb
202
264
  - spec/integration/sql_retry/lock_wait_spec.rb
203
265
  - spec/integration/sql_retry/lock_wait_timeout_test_helper.rb
266
+ - spec/integration/sql_retry/proxysql_helper.rb
267
+ - spec/integration/sql_retry/retry_with_proxysql_spec.rb
204
268
  - spec/integration/table_spec.rb
269
+ - spec/integration/toxiproxy_helper.rb
205
270
  - spec/test_helper.rb
206
271
  - spec/unit/atomic_switcher_spec.rb
207
272
  - spec/unit/chunk_finder_spec.rb
@@ -1,6 +0,0 @@
1
- source "https://rubygems.org"
2
-
3
- gem "mysql", "~> 2.8.1"
4
- gem "activerecord", "~> 2.3.18"
5
- gem "iconv", "~> 1.0.4"
6
- gemspec :path=>"../"
@@ -1,5 +0,0 @@
1
- source "https://rubygems.org"
2
-
3
- gem "mysql", "~> 2.8.1"
4
- gem "activerecord", "~> 3.2.2"
5
- gemspec :path=>"../"
@@ -1,5 +0,0 @@
1
- source "https://rubygems.org"
2
-
3
- gem "mysql2", "~> 0.3.11"
4
- gem "activerecord", "~> 3.2.2"
5
- gemspec :path=>"../"
@@ -1,5 +0,0 @@
1
- source "https://rubygems.org"
2
-
3
- gem "mysql2", "~> 0.3.17"
4
- gem "activerecord", "~> 4.0.13"
5
- gemspec :path=>"../"
@@ -1,5 +0,0 @@
1
- source "https://rubygems.org"
2
-
3
- gem "mysql2", "~> 0.3.17"
4
- gem "activerecord", "~> 4.1.9"
5
- gemspec :path=>"../"
@@ -1,5 +0,0 @@
1
- source "https://rubygems.org"
2
-
3
- gem "mysql2", "~> 0.3.17"
4
- gem "activerecord", "~> 4.2.0"
5
- gemspec :path=>"../"
@@ -1,5 +0,0 @@
1
- source "https://rubygems.org"
2
-
3
- gem "mysql2", "~> 0.4.5"
4
- gem "activerecord", "~> 5.0.2"
5
- gemspec :path=>"../"