lhm-shopify 3.5.5 → 4.1.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (63) hide show
  1. checksums.yaml +4 -4
  2. data/.github/workflows/test.yml +20 -18
  3. data/Appraisals +8 -19
  4. data/CHANGELOG.md +16 -0
  5. data/Gemfile.lock +37 -20
  6. data/README.md +21 -14
  7. data/dev.yml +12 -8
  8. data/docker-compose-mysql-5.7.yml +1 -0
  9. data/docker-compose-mysql-8.0.yml +63 -0
  10. data/docker-compose.yml +5 -3
  11. data/gemfiles/activerecord_6.1.gemfile +1 -0
  12. data/gemfiles/activerecord_6.1.gemfile.lock +23 -13
  13. data/gemfiles/{activerecord_7.0.0.alpha2.gemfile → activerecord_7.0.gemfile} +2 -1
  14. data/gemfiles/{activerecord_7.0.0.alpha2.gemfile.lock → activerecord_7.0.gemfile.lock} +29 -19
  15. data/gemfiles/{activerecord_6.0.gemfile → activerecord_7.1.gemfile} +1 -1
  16. data/gemfiles/activerecord_7.1.gemfile.lock +83 -0
  17. data/lhm.gemspec +2 -1
  18. data/lib/lhm/atomic_switcher.rb +3 -3
  19. data/lib/lhm/chunker.rb +4 -4
  20. data/lib/lhm/connection.rb +9 -1
  21. data/lib/lhm/sql_helper.rb +1 -1
  22. data/lib/lhm/sql_retry.rb +36 -18
  23. data/lib/lhm/table.rb +3 -4
  24. data/lib/lhm/throttler/replica_lag.rb +166 -0
  25. data/lib/lhm/throttler/slave_lag.rb +5 -155
  26. data/lib/lhm/throttler/threads_running.rb +3 -1
  27. data/lib/lhm/throttler.rb +7 -3
  28. data/lib/lhm/version.rb +1 -1
  29. data/scripts/helpers/wait-for-dbs.sh +3 -3
  30. data/scripts/mysql/writer/create_users.sql +1 -1
  31. data/spec/.lhm.example +1 -1
  32. data/spec/README.md +8 -9
  33. data/spec/integration/atomic_switcher_spec.rb +6 -10
  34. data/spec/integration/chunk_insert_spec.rb +2 -2
  35. data/spec/integration/chunker_spec.rb +54 -44
  36. data/spec/integration/database.yml +4 -4
  37. data/spec/integration/entangler_spec.rb +4 -4
  38. data/spec/integration/integration_helper.rb +23 -15
  39. data/spec/integration/lhm_spec.rb +70 -44
  40. data/spec/integration/locked_switcher_spec.rb +2 -2
  41. data/spec/integration/proxysql_spec.rb +10 -10
  42. data/spec/integration/sql_retry/db_connection_helper.rb +2 -4
  43. data/spec/integration/sql_retry/lock_wait_spec.rb +7 -8
  44. data/spec/integration/sql_retry/lock_wait_timeout_test_helper.rb +18 -10
  45. data/spec/integration/sql_retry/proxysql_helper.rb +1 -1
  46. data/spec/integration/sql_retry/retry_with_proxysql_spec.rb +1 -2
  47. data/spec/integration/table_spec.rb +1 -1
  48. data/spec/integration/toxiproxy_helper.rb +1 -1
  49. data/spec/test_helper.rb +27 -3
  50. data/spec/unit/atomic_switcher_spec.rb +2 -2
  51. data/spec/unit/chunker_spec.rb +43 -43
  52. data/spec/unit/connection_spec.rb +2 -2
  53. data/spec/unit/entangler_spec.rb +14 -24
  54. data/spec/unit/printer_spec.rb +2 -6
  55. data/spec/unit/sql_helper_spec.rb +2 -2
  56. data/spec/unit/throttler/{slave_lag_spec.rb → replica_lag_spec.rb} +84 -92
  57. data/spec/unit/throttler/threads_running_spec.rb +18 -0
  58. data/spec/unit/throttler_spec.rb +8 -8
  59. metadata +26 -12
  60. data/.travis.yml +0 -21
  61. data/gemfiles/activerecord_5.2.gemfile +0 -9
  62. data/gemfiles/activerecord_5.2.gemfile.lock +0 -65
  63. data/gemfiles/activerecord_6.0.gemfile.lock +0 -67
@@ -1,161 +1,11 @@
1
+ require 'lhm/throttler/replica_lag'
2
+
1
3
  module Lhm
2
4
  module Throttler
3
-
4
- def self.format_hosts(hosts)
5
- formatted_hosts = []
6
- hosts.each do |host|
7
- if host && !host.match(/localhost/) && !host.match(/127.0.0.1/)
8
- formatted_hosts << host.partition(':')[0]
9
- end
10
- end
11
- formatted_hosts
12
- end
13
-
14
- class SlaveLag
15
- include Command
16
-
17
- INITIAL_TIMEOUT = 0.1
18
- DEFAULT_STRIDE = 2_000
19
- DEFAULT_MAX_ALLOWED_LAG = 10
20
-
21
- MAX_TIMEOUT = INITIAL_TIMEOUT * 1024
22
-
23
- attr_accessor :timeout_seconds, :allowed_lag, :stride, :connection
24
-
5
+ class SlaveLag < ReplicaLag
25
6
  def initialize(options = {})
26
- @timeout_seconds = INITIAL_TIMEOUT
27
- @stride = options[:stride] || DEFAULT_STRIDE
28
- @allowed_lag = options[:allowed_lag] || DEFAULT_MAX_ALLOWED_LAG
29
- @slaves = {}
30
- @get_config = options[:current_config]
31
- @check_only = options[:check_only]
32
- end
33
-
34
- def execute
35
- sleep(throttle_seconds)
36
- end
37
-
38
- private
39
-
40
- def throttle_seconds
41
- lag = max_current_slave_lag
42
-
43
- if lag > @allowed_lag && @timeout_seconds < MAX_TIMEOUT
44
- Lhm.logger.info("Increasing timeout between strides from #{@timeout_seconds} to #{@timeout_seconds * 2} because #{lag} seconds of slave lag detected is greater than the maximum of #{@allowed_lag} seconds allowed.")
45
- @timeout_seconds = @timeout_seconds * 2
46
- elsif lag <= @allowed_lag && @timeout_seconds > INITIAL_TIMEOUT
47
- Lhm.logger.info("Decreasing timeout between strides from #{@timeout_seconds} to #{@timeout_seconds / 2} because #{lag} seconds of slave lag detected is less than or equal to the #{@allowed_lag} seconds allowed.")
48
- @timeout_seconds = @timeout_seconds / 2
49
- else
50
- @timeout_seconds
51
- end
52
- end
53
-
54
- def slaves
55
- @slaves[@connection] ||= get_slaves
56
- end
57
-
58
- def get_slaves
59
- slaves = []
60
- if @check_only.nil? or !@check_only.respond_to?(:call)
61
- slave_hosts = master_slave_hosts
62
- while slave_hosts.any? do
63
- host = slave_hosts.pop
64
- slave = Slave.new(host, @get_config)
65
- if !slaves.map(&:host).include?(host) && slave.connection
66
- slaves << slave
67
- slave_hosts.concat(slave.slave_hosts)
68
- end
69
- end
70
- else
71
- slave_config = @check_only.call
72
- slaves << Slave.new(slave_config['host'], @get_config)
73
- end
74
- slaves
75
- end
76
-
77
- def master_slave_hosts
78
- Throttler.format_hosts(@connection.select_values(Slave::SQL_SELECT_SLAVE_HOSTS))
79
- end
80
-
81
- def max_current_slave_lag
82
- max = slaves.map { |slave| slave.lag }.push(0).max
83
- Lhm.logger.info "Max current slave lag: #{max}"
84
- max
85
- end
86
- end
87
-
88
- class Slave
89
- SQL_SELECT_SLAVE_HOSTS = "SELECT host FROM information_schema.processlist WHERE command LIKE 'Binlog Dump%'"
90
- SQL_SELECT_MAX_SLAVE_LAG = 'SHOW SLAVE STATUS'
91
-
92
- attr_reader :host, :connection
93
-
94
- def initialize(host, connection_config = nil)
95
- @host = host
96
- @connection_config = prepare_connection_config(connection_config)
97
- @connection = client(@connection_config)
98
- end
99
-
100
- def slave_hosts
101
- Throttler.format_hosts(query_connection(SQL_SELECT_SLAVE_HOSTS, 'host'))
102
- end
103
-
104
- def lag
105
- query_connection(SQL_SELECT_MAX_SLAVE_LAG, 'Seconds_Behind_Master').first.to_i
106
- end
107
-
108
- private
109
-
110
- def client(config)
111
- begin
112
- Lhm.logger.info "Connecting to #{@host} on database: #{config[:database]}"
113
- Mysql2::Client.new(config)
114
- rescue Mysql2::Error => e
115
- Lhm.logger.info "Error connecting to #{@host}: #{e}"
116
- nil
117
- end
118
- end
119
-
120
- def prepare_connection_config(config_proc)
121
- config = if config_proc
122
- if config_proc.respond_to?(:call) # if we get a proc
123
- config_proc.call
124
- else
125
- raise ArgumentError, "Expected #{config_proc.inspect} to respond to `call`"
126
- end
127
- else
128
- db_config
129
- end
130
- config.deep_symbolize_keys!
131
- config[:host] = @host
132
- config
133
- end
134
-
135
- def query_connection(query, result)
136
- begin
137
- @connection.query(query).map { |row| row[result] }
138
- rescue Mysql2::Error => e
139
- Lhm.logger.info "Unable to connect and/or query #{host}: #{e}"
140
- [nil]
141
- end
142
- end
143
-
144
- private
145
-
146
- def db_config
147
- if ar_supports_db_config?
148
- ActiveRecord::Base.connection_pool.db_config.configuration_hash.dup
149
- else
150
- ActiveRecord::Base.connection_pool.spec.config.dup
151
- end
152
- end
153
-
154
- def ar_supports_db_config?
155
- # https://api.rubyonrails.org/v6.0/classes/ActiveRecord/ConnectionAdapters/ConnectionPool.html <-- has spec
156
- # vs
157
- # https://api.rubyonrails.org/v6.1/classes/ActiveRecord/ConnectionAdapters/ConnectionPool.html <-- has db_config
158
- ActiveRecord::VERSION::MAJOR > 6 || ActiveRecord::VERSION::MAJOR == 6 && ActiveRecord::VERSION::MINOR >= 1
7
+ Lhm.logger.warn("Class `SlaveLag` is deprecated. Use `ReplicaLag` class instead.")
8
+ super(options)
159
9
  end
160
10
  end
161
11
  end
@@ -3,14 +3,16 @@ module Lhm
3
3
  class ThreadsRunning
4
4
  include Command
5
5
 
6
+ DEFAULT_STRIDE = 2_000
6
7
  DEFAULT_INITIAL_TIMEOUT = 0.1
7
8
  DEFAULT_HEALTHY_RANGE = (0..50)
8
9
 
9
- attr_accessor :timeout_seconds, :healthy_range, :connection
10
+ attr_accessor :timeout_seconds, :healthy_range, :connection, :stride
10
11
  attr_reader :max_timeout_seconds, :initial_timeout_seconds
11
12
 
12
13
  def initialize(options = {})
13
14
  @initial_timeout_seconds = options[:initial_timeout] || DEFAULT_INITIAL_TIMEOUT
15
+ @stride = options[:stride] || DEFAULT_STRIDE
14
16
  @max_timeout_seconds = options[:max_timeout] || (@initial_timeout_seconds * 1024)
15
17
  @timeout_seconds = @initial_timeout_seconds
16
18
  @healthy_range = options[:healthy_range] || DEFAULT_HEALTHY_RANGE
data/lib/lhm/throttler.rb CHANGED
@@ -1,12 +1,16 @@
1
1
  require 'lhm/throttler/time'
2
+ require 'lhm/throttler/replica_lag'
2
3
  require 'lhm/throttler/slave_lag'
3
4
  require 'lhm/throttler/threads_running'
4
5
 
5
6
  module Lhm
6
7
  module Throttler
7
- CLASSES = { :time_throttler => Throttler::Time,
8
- :slave_lag_throttler => Throttler::SlaveLag,
9
- :threads_running_throttler => Throttler::ThreadsRunning }
8
+ CLASSES = {
9
+ :time_throttler => Throttler::Time,
10
+ :replica_lag_throttler => Throttler::ReplicaLag,
11
+ :slave_lag_throttler => Throttler::SlaveLag,
12
+ :threads_running_throttler => Throttler::ThreadsRunning
13
+ }
10
14
 
11
15
  def throttler
12
16
  @throttler ||= Throttler::Time.new
data/lib/lhm/version.rb CHANGED
@@ -2,5 +2,5 @@
2
2
  # Schmidt
3
3
 
4
4
  module Lhm
5
- VERSION = '3.5.5'
5
+ VERSION = '4.1.0'
6
6
  end
@@ -1,19 +1,19 @@
1
1
  #!/bin/bash
2
2
  # Wait for writer
3
3
  echo "Waiting for MySQL-1: "
4
- while ! (mysqladmin ping --host="127.0.0.1" --port=33006 --user=root --password=password --silent 2> /dev/null); do
4
+ while ! (mysqladmin ping --host="127.0.0.1" --port=13006 --user=root --password=password --silent 2> /dev/null); do
5
5
  echo -ne "."
6
6
  sleep 1
7
7
  done
8
8
  # Wait for reader
9
9
  echo "Waiting for MySQL-2: "
10
- while ! (mysqladmin ping --host="127.0.0.1" --port=33007 --user=root --password=password --silent 2> /dev/null); do
10
+ while ! (mysqladmin ping --host="127.0.0.1" --port=13007 --user=root --password=password --silent 2> /dev/null); do
11
11
  echo -ne "."
12
12
  sleep 1
13
13
  done
14
14
  # Wait for proxysql
15
15
  echo "Waiting for ProxySQL:"
16
- while ! (mysqladmin ping --host="127.0.0.1" --port=33005 --user=root --password=password --silent 2> /dev/null); do
16
+ while ! (mysqladmin ping --host="127.0.0.1" --port=13005 --user=root --password=password --silent 2> /dev/null); do
17
17
  echo -ne "."
18
18
  sleep 1
19
19
  done
@@ -3,4 +3,4 @@ CREATE USER IF NOT EXISTS 'writer'@'%' IDENTIFIED BY 'password';
3
3
  CREATE USER IF NOT EXISTS 'reader'@'%' IDENTIFIED BY 'password';
4
4
 
5
5
  CREATE USER IF NOT EXISTS 'replication'@'%' IDENTIFIED BY 'password';
6
- GRANT REPLICATION SLAVE ON *.* TO' replication'@'%' IDENTIFIED BY 'password';
6
+ GRANT REPLICATION SLAVE ON *.* TO' replication'@'%';
data/spec/.lhm.example CHANGED
@@ -1,4 +1,4 @@
1
1
  mysqldir=/usr/local/mysql
2
2
  basedir=~/lhm-cluster
3
3
  master_port=3306
4
- slave_port=3307
4
+ replica_port=3307
data/spec/README.md CHANGED
@@ -1,4 +1,4 @@
1
- # Preparing for master slave integration tests
1
+ # Preparing for master replica integration tests
2
2
 
3
3
  ## Configuration
4
4
 
@@ -7,10 +7,10 @@ create ~/.lhm:
7
7
  mysqldir=/usr/local/mysql
8
8
  basedir=~/lhm-cluster
9
9
  master_port=3306
10
- slave_port=3307
10
+ replica_port=3307
11
11
 
12
12
  mysqldir specifies the location of your mysql install. basedir is the
13
- directory master and slave databases will get installed into.
13
+ directory master and replica databases will get installed into.
14
14
 
15
15
  ## Automatic setup
16
16
 
@@ -18,8 +18,8 @@ directory master and slave databases will get installed into.
18
18
 
19
19
  bin/lhm-spec-clobber.sh
20
20
 
21
- You can set the integration specs up to run against a master slave setup by
22
- running the included that. This deletes the configured lhm master slave setup and reinstalls and configures a master slave setup.
21
+ You can set the integration specs up to run against a master replica setup by
22
+ running the included that. This deletes the configured lhm master replica setup and reinstalls and configures a master replica setup.
23
23
 
24
24
  Follow the manual instructions if you want more control over this process.
25
25
 
@@ -33,7 +33,7 @@ Follow the manual instructions if you want more control over this process.
33
33
 
34
34
  basedir=/opt/lhm-luster
35
35
  mysqld --defaults-file="$basedir/master/my.cnf"
36
- mysqld --defaults-file="$basedir/slave/my.cnf"
36
+ mysqld --defaults-file="$basedir/replica/my.cnf"
37
37
 
38
38
  ### run the grants
39
39
 
@@ -46,13 +46,12 @@ Setup the dependency gems
46
46
  export BUNDLE_GEMFILE=gemfiles/ar-4.2_mysql2.gemfile
47
47
  bundle install
48
48
 
49
- To run specs in slave mode, set the MASTER_SLAVE=1 when running tests:
49
+ To run specs in replica mode, set the MASTER_REPLICA=1 when running tests:
50
50
 
51
- MASTER_SLAVE=1 bundle exec rake specs
51
+ MASTER_REPLICA=1 bundle exec rake specs
52
52
 
53
53
  # connecting
54
54
 
55
55
  you can connect by running (with the respective ports):
56
56
 
57
57
  mysql --protocol=TCP -p3307
58
-
@@ -33,8 +33,8 @@ describe Lhm::AtomicSwitcher do
33
33
  ar_connection = mock()
34
34
  ar_connection.stubs(:data_source_exists?).returns(true)
35
35
  ar_connection.stubs(:active?).returns(true)
36
- ar_connection.stubs(:execute).returns([["dummy"]], [["dummy"]], [["dummy"]])
37
- .then
36
+ ar_connection.stubs(:select_value).returns("dummy")
37
+ ar_connection.stubs(:execute)
38
38
  .raises(ActiveRecord::StatementInvalid, 'Lock wait timeout exceeded; try restarting transaction.')
39
39
  .then
40
40
  .returns([["dummy"]]) # Matches initial host -> triggers retry
@@ -62,16 +62,12 @@ describe Lhm::AtomicSwitcher do
62
62
  ar_connection = mock()
63
63
  ar_connection.stubs(:data_source_exists?).returns(true)
64
64
  ar_connection.stubs(:active?).returns(true)
65
- ar_connection.stubs(:execute).returns([["dummy"]], [["dummy"]], [["dummy"]])
66
- .then
65
+ ar_connection.stubs(:select_value).returns("dummy")
66
+ ar_connection.stubs(:execute)
67
67
  .raises(ActiveRecord::StatementInvalid, 'Lock wait timeout exceeded; try restarting transaction.')
68
68
  .then
69
- .returns([["dummy"]]) # triggers retry 1
70
- .then
71
69
  .raises(ActiveRecord::StatementInvalid, 'Lock wait timeout exceeded; try restarting transaction.')
72
70
  .then
73
- .returns([["dummy"]]) # triggers retry 2
74
- .then
75
71
  .raises(ActiveRecord::StatementInvalid, 'Lock wait timeout exceeded; try restarting transaction.') # triggers retry 2
76
72
 
77
73
  connection = Lhm::Connection.new(connection: ar_connection, options: {
@@ -110,7 +106,7 @@ describe Lhm::AtomicSwitcher do
110
106
  switcher = Lhm::AtomicSwitcher.new(@migration, connection)
111
107
  switcher.run
112
108
 
113
- slave do
109
+ replica do
114
110
  value(data_source_exists?(@origin)).must_equal true
115
111
  value(table_read(@migration.archive_name).columns.keys).must_include 'origin'
116
112
  end
@@ -120,7 +116,7 @@ describe Lhm::AtomicSwitcher do
120
116
  switcher = Lhm::AtomicSwitcher.new(@migration, connection)
121
117
  switcher.run
122
118
 
123
- slave do
119
+ replica do
124
120
  value(data_source_exists?(@destination)).must_equal false
125
121
  value(table_read(@origin.name).columns.keys).must_include 'destination'
126
122
  end
@@ -19,10 +19,10 @@ describe Lhm::ChunkInsert do
19
19
  assert_equal 1, @instance.insert_and_return_count_of_rows_created
20
20
  end
21
21
 
22
- it "inserts the record into the slave" do
22
+ it "inserts the record into the replica" do
23
23
  @instance.insert_and_return_count_of_rows_created
24
24
 
25
- slave do
25
+ replica do
26
26
  value(count_all(@destination.name)).must_equal(1)
27
27
  end
28
28
  end
@@ -28,7 +28,7 @@ describe Lhm::Chunker do
28
28
 
29
29
  Lhm::Chunker.new(@migration, connection, {throttler: throttler, printer: printer} ).run
30
30
 
31
- slave do
31
+ replica do
32
32
  value(count_all(@destination.name)).must_equal(1)
33
33
  end
34
34
 
@@ -41,7 +41,7 @@ describe Lhm::Chunker do
41
41
 
42
42
  Lhm::Chunker.new(@migration, connection, {throttler: throttler, printer: printer} ).run
43
43
 
44
- slave do
44
+ replica do
45
45
  value(count_all(@destination.name)).must_equal(2)
46
46
  end
47
47
  end
@@ -57,7 +57,7 @@ describe Lhm::Chunker do
57
57
 
58
58
  Lhm::Chunker.new(migration, connection, {throttler: throttler, printer: printer} ).run
59
59
 
60
- slave do
60
+ replica do
61
61
  value(count_all(destination.name)).must_equal(2)
62
62
  end
63
63
  end
@@ -74,7 +74,8 @@ describe Lhm::Chunker do
74
74
  Lhm::Chunker.new(migration, connection, {raise_on_warnings: true, throttler: throttler, printer: printer} ).run
75
75
  end
76
76
 
77
- assert_match "Unexpected warning found for inserted row: Duplicate entry '1001' for key 'index_custom_primary_key_on_id'", exception.message
77
+ error_key = index_key("custom_primary_key_dest", "index_custom_primary_key_on_id")
78
+ assert_match "Unexpected warning found for inserted row: Duplicate entry '1001' for key '#{error_key}'", exception.message
78
79
  end
79
80
 
80
81
  it 'should copy and warn on unexpected warnings by default' do
@@ -87,8 +88,10 @@ describe Lhm::Chunker do
87
88
 
88
89
  Lhm::Chunker.new(migration, connection, {throttler: throttler, printer: printer} ).run
89
90
 
91
+ error_key = index_key("custom_primary_key_dest", "index_custom_primary_key_on_id")
92
+
90
93
  assert_equal 2, log_messages.length
91
- assert log_messages[1].include?("Unexpected warning found for inserted row: Duplicate entry '1001' for key 'index_custom_primary_key_on_id'"), log_messages
94
+ assert log_messages[1].include?("Unexpected warning found for inserted row: Duplicate entry '1001' for key '#{error_key}'"), log_messages
92
95
  end
93
96
 
94
97
  it 'should log two times for two unexpected warnings' do
@@ -103,9 +106,11 @@ describe Lhm::Chunker do
103
106
 
104
107
  Lhm::Chunker.new(migration, connection, {throttler: throttler, printer: printer} ).run
105
108
 
109
+ error_key = index_key("custom_primary_key_dest", "index_custom_primary_key_on_id")
110
+
106
111
  assert_equal 3, log_messages.length
107
- assert log_messages[1].include?("Unexpected warning found for inserted row: Duplicate entry '1001' for key 'index_custom_primary_key_on_id'"), log_messages
108
- assert log_messages[2].include?("Unexpected warning found for inserted row: Duplicate entry '1002' for key 'index_custom_primary_key_on_id'"), log_messages
112
+ assert log_messages[1].include?("Unexpected warning found for inserted row: Duplicate entry '1001' for key '#{error_key}'"), log_messages
113
+ assert log_messages[2].include?("Unexpected warning found for inserted row: Duplicate entry '1002' for key '#{error_key}'"), log_messages
109
114
  end
110
115
 
111
116
  it 'should copy and warn on unexpected warnings' do
@@ -118,8 +123,10 @@ describe Lhm::Chunker do
118
123
 
119
124
  Lhm::Chunker.new(migration, connection, {raise_on_warnings: false, throttler: throttler, printer: printer} ).run
120
125
 
126
+ error_key = index_key("custom_primary_key_dest", "index_custom_primary_key_on_id")
127
+
121
128
  assert_equal 2, log_messages.length
122
- assert log_messages[1].include?("Unexpected warning found for inserted row: Duplicate entry '1001' for key 'index_custom_primary_key_on_id'"), log_messages
129
+ assert log_messages[1].include?("Unexpected warning found for inserted row: Duplicate entry '1001' for key '#{error_key}'"), log_messages
123
130
  end
124
131
 
125
132
  it 'should create the modified destination, even if the source is empty' do
@@ -127,7 +134,7 @@ describe Lhm::Chunker do
127
134
 
128
135
  Lhm::Chunker.new(@migration, connection, {throttler: throttler, printer: printer} ).run
129
136
 
130
- slave do
137
+ replica do
131
138
  value(count_all(@destination.name)).must_equal(0)
132
139
  end
133
140
 
@@ -136,20 +143,17 @@ describe Lhm::Chunker do
136
143
  it 'should copy 23 rows from origin to destination in one shot, regardless of the value of the id' do
137
144
  23.times { |n| execute("insert into origin set id = '#{ n * n + 23 }'") }
138
145
 
139
- printer = MiniTest::Mock.new
140
- printer.expect(:notify, :return_value, [Integer, Integer])
141
- printer.expect(:end, :return_value, [])
146
+ printer = mock("printer")
147
+ printer.expects(:notify).with(kind_of(Integer), kind_of(Integer))
148
+ printer.expects(:end)
142
149
 
143
150
  Lhm::Chunker.new(
144
151
  @migration, connection, { throttler: throttler, printer: printer }
145
152
  ).run
146
153
 
147
- slave do
154
+ replica do
148
155
  value(count_all(@destination.name)).must_equal(23)
149
156
  end
150
-
151
- printer.verify
152
-
153
157
  end
154
158
 
155
159
  it 'should copy all the records of a table, even if the last chunk starts with the last record of it.' do
@@ -160,42 +164,40 @@ describe Lhm::Chunker do
160
164
  @migration, connection, { throttler: Lhm::Throttler::Time.new(stride: 10), printer: printer }
161
165
  ).run
162
166
 
163
- slave do
167
+ replica do
164
168
  value(count_all(@destination.name)).must_equal(11)
165
169
  end
166
170
 
167
171
  end
168
172
 
169
- it 'should copy 23 rows from origin to destination in one shot with slave lag based throttler, regardless of the value of the id' do
173
+ it 'should copy 23 rows from origin to destination in one shot with replica lag based throttler, regardless of the value of the id' do
170
174
  23.times { |n| execute("insert into origin set id = '#{ 100000 + n * n + 23 }'") }
171
175
 
172
- printer = MiniTest::Mock.new
173
- printer.expect(:notify, :return_value, [Integer, Integer])
174
- printer.expect(:end, :return_value, [])
176
+ printer = mock("printer")
177
+ printer.expects(:notify).with(kind_of(Integer), kind_of(Integer))
178
+ printer.expects(:end)
175
179
 
176
- Lhm::Throttler::Slave.any_instance.stubs(:slave_hosts).returns(['127.0.0.1'])
177
- Lhm::Throttler::SlaveLag.any_instance.stubs(:master_slave_hosts).returns(['127.0.0.1'])
180
+ Lhm::Throttler::Replica.any_instance.stubs(:replica_hosts).returns(['127.0.0.1'])
181
+ Lhm::Throttler::ReplicaLag.any_instance.stubs(:master_replica_hosts).returns(['127.0.0.1'])
178
182
 
179
183
  Lhm::Chunker.new(
180
- @migration, connection, { throttler: Lhm::Throttler::SlaveLag.new(stride: 100), printer: printer }
184
+ @migration, connection, { throttler: Lhm::Throttler::ReplicaLag.new(stride: 100), printer: printer }
181
185
  ).run
182
186
 
183
- slave do
187
+ replica do
184
188
  value(count_all(@destination.name)).must_equal(23)
185
189
  end
186
-
187
- printer.verify
188
190
  end
189
191
 
190
- it 'should throttle work stride based on slave lag' do
192
+ it 'should throttle work stride based on replica lag' do
191
193
  15.times { |n| execute("insert into origin set id = '#{ (n * n) + 1 }'") }
192
194
 
193
195
  printer = mock()
194
196
  printer.expects(:notify).with(instance_of(Integer), instance_of(Integer)).twice
195
197
  printer.expects(:end)
196
198
 
197
- throttler = Lhm::Throttler::SlaveLag.new(stride: 10, allowed_lag: 0)
198
- def throttler.max_current_slave_lag
199
+ throttler = Lhm::Throttler::ReplicaLag.new(stride: 10, allowed_lag: 0)
200
+ def throttler.max_current_replica_lag
199
201
  1
200
202
  end
201
203
 
@@ -203,14 +205,14 @@ describe Lhm::Chunker do
203
205
  @migration, connection, { throttler: throttler, printer: printer }
204
206
  ).run
205
207
 
206
- assert_equal(Lhm::Throttler::SlaveLag::INITIAL_TIMEOUT * 2 * 2, throttler.timeout_seconds)
208
+ assert_equal(Lhm::Throttler::ReplicaLag::INITIAL_TIMEOUT * 2 * 2, throttler.timeout_seconds)
207
209
 
208
- slave do
210
+ replica do
209
211
  value(count_all(@destination.name)).must_equal(15)
210
212
  end
211
213
  end
212
214
 
213
- it 'should detect a single slave with no lag in the default configuration' do
215
+ it 'should detect a single replica with no lag in the default configuration' do
214
216
  15.times { |n| execute("insert into origin set id = '#{ (n * n) + 1 }'") }
215
217
 
216
218
  printer = mock()
@@ -218,16 +220,16 @@ describe Lhm::Chunker do
218
220
  printer.expects(:verify)
219
221
  printer.expects(:end)
220
222
 
221
- Lhm::Throttler::Slave.any_instance.stubs(:slave_hosts).returns(['127.0.0.1'])
222
- Lhm::Throttler::SlaveLag.any_instance.stubs(:master_slave_hosts).returns(['127.0.0.1'])
223
+ Lhm::Throttler::Replica.any_instance.stubs(:replica_hosts).returns(['127.0.0.1'])
224
+ Lhm::Throttler::ReplicaLag.any_instance.stubs(:master_replica_hosts).returns(['127.0.0.1'])
223
225
 
224
- throttler = Lhm::Throttler::SlaveLag.new(stride: 10, allowed_lag: 0)
226
+ throttler = Lhm::Throttler::ReplicaLag.new(stride: 10, allowed_lag: 0)
225
227
 
226
- if master_slave_mode?
227
- def throttler.slave_connection(slave)
228
+ if master_replica_mode?
229
+ def throttler.replica_connection(replica)
228
230
  config = ActiveRecord::Base.connection_pool.db_config.configuration_hash.dup
229
- config[:host] = slave
230
- config[:port] = 33007
231
+ config[:host] = replica
232
+ config[:port] = 13007
231
233
  ActiveRecord::Base.send('mysql2_connection', config)
232
234
  end
233
235
  end
@@ -236,10 +238,10 @@ describe Lhm::Chunker do
236
238
  @migration, connection, { throttler: throttler, printer: printer }
237
239
  ).run
238
240
 
239
- assert_equal(Lhm::Throttler::SlaveLag::INITIAL_TIMEOUT, throttler.timeout_seconds)
240
- assert_equal(0, throttler.send(:max_current_slave_lag))
241
+ assert_equal(Lhm::Throttler::ReplicaLag::INITIAL_TIMEOUT, throttler.timeout_seconds)
242
+ assert_equal(0, throttler.send(:max_current_replica_lag))
241
243
 
242
- slave do
244
+ replica do
243
245
  value(count_all(@destination.name)).must_equal(15)
244
246
  end
245
247
 
@@ -261,9 +263,17 @@ describe Lhm::Chunker do
261
263
 
262
264
  assert_match "Verification failed, aborting early", exception.message
263
265
 
264
- slave do
266
+ replica do
265
267
  value(count_all(@destination.name)).must_equal(0)
266
268
  end
267
269
  end
268
270
  end
271
+
272
+ def index_key(table_name, index_name)
273
+ if mysql_version.start_with?("8")
274
+ "#{table_name}.#{index_name}"
275
+ else
276
+ index_name
277
+ end
278
+ end
269
279
  end
@@ -2,17 +2,17 @@ master:
2
2
  host: mysql-1
3
3
  user: root
4
4
  password: password
5
- port: 33006
6
- slave:
5
+ port: 13006
6
+ replica:
7
7
  host: mysql-2
8
8
  user: root
9
9
  password: password
10
- port: 33007
10
+ port: 13007
11
11
  proxysql:
12
12
  host: proxysql
13
13
  user: root
14
14
  password: password
15
- port: 33005
15
+ port: 13005
16
16
  master_toxic:
17
17
  host: toxiproxy
18
18
  user: root
@@ -27,7 +27,7 @@ describe Lhm::Entangler do
27
27
  execute("insert into origin (common) values ('inserted')")
28
28
  end
29
29
 
30
- slave do
30
+ replica do
31
31
  value(count(:destination, 'common', 'inserted')).must_equal(1)
32
32
  end
33
33
  end
@@ -39,7 +39,7 @@ describe Lhm::Entangler do
39
39
  execute("delete from origin where common = 'inserted'")
40
40
  end
41
41
 
42
- slave do
42
+ replica do
43
43
  value(count(:destination, 'common', 'inserted')).must_equal(0)
44
44
  end
45
45
  end
@@ -50,7 +50,7 @@ describe Lhm::Entangler do
50
50
  execute("update origin set common = 'updated'")
51
51
  end
52
52
 
53
- slave do
53
+ replica do
54
54
  value(count(:destination, 'common', 'updated')).must_equal(1)
55
55
  end
56
56
  end
@@ -60,7 +60,7 @@ describe Lhm::Entangler do
60
60
 
61
61
  execute("insert into origin (common) values ('inserted')")
62
62
 
63
- slave do
63
+ replica do
64
64
  value(count(:destination, 'common', 'inserted')).must_equal(0)
65
65
  end
66
66
  end