lhm-shopify 3.5.5 → 4.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (63) hide show
  1. checksums.yaml +4 -4
  2. data/.github/workflows/test.yml +20 -18
  3. data/Appraisals +8 -19
  4. data/CHANGELOG.md +16 -0
  5. data/Gemfile.lock +37 -20
  6. data/README.md +21 -14
  7. data/dev.yml +12 -8
  8. data/docker-compose-mysql-5.7.yml +1 -0
  9. data/docker-compose-mysql-8.0.yml +63 -0
  10. data/docker-compose.yml +5 -3
  11. data/gemfiles/activerecord_6.1.gemfile +1 -0
  12. data/gemfiles/activerecord_6.1.gemfile.lock +23 -13
  13. data/gemfiles/{activerecord_7.0.0.alpha2.gemfile → activerecord_7.0.gemfile} +2 -1
  14. data/gemfiles/{activerecord_7.0.0.alpha2.gemfile.lock → activerecord_7.0.gemfile.lock} +29 -19
  15. data/gemfiles/{activerecord_6.0.gemfile → activerecord_7.1.gemfile} +1 -1
  16. data/gemfiles/activerecord_7.1.gemfile.lock +83 -0
  17. data/lhm.gemspec +2 -1
  18. data/lib/lhm/atomic_switcher.rb +3 -3
  19. data/lib/lhm/chunker.rb +4 -4
  20. data/lib/lhm/connection.rb +9 -1
  21. data/lib/lhm/sql_helper.rb +1 -1
  22. data/lib/lhm/sql_retry.rb +36 -18
  23. data/lib/lhm/table.rb +3 -4
  24. data/lib/lhm/throttler/replica_lag.rb +166 -0
  25. data/lib/lhm/throttler/slave_lag.rb +5 -155
  26. data/lib/lhm/throttler/threads_running.rb +3 -1
  27. data/lib/lhm/throttler.rb +7 -3
  28. data/lib/lhm/version.rb +1 -1
  29. data/scripts/helpers/wait-for-dbs.sh +3 -3
  30. data/scripts/mysql/writer/create_users.sql +1 -1
  31. data/spec/.lhm.example +1 -1
  32. data/spec/README.md +8 -9
  33. data/spec/integration/atomic_switcher_spec.rb +6 -10
  34. data/spec/integration/chunk_insert_spec.rb +2 -2
  35. data/spec/integration/chunker_spec.rb +54 -44
  36. data/spec/integration/database.yml +4 -4
  37. data/spec/integration/entangler_spec.rb +4 -4
  38. data/spec/integration/integration_helper.rb +23 -15
  39. data/spec/integration/lhm_spec.rb +70 -44
  40. data/spec/integration/locked_switcher_spec.rb +2 -2
  41. data/spec/integration/proxysql_spec.rb +10 -10
  42. data/spec/integration/sql_retry/db_connection_helper.rb +2 -4
  43. data/spec/integration/sql_retry/lock_wait_spec.rb +7 -8
  44. data/spec/integration/sql_retry/lock_wait_timeout_test_helper.rb +18 -10
  45. data/spec/integration/sql_retry/proxysql_helper.rb +1 -1
  46. data/spec/integration/sql_retry/retry_with_proxysql_spec.rb +1 -2
  47. data/spec/integration/table_spec.rb +1 -1
  48. data/spec/integration/toxiproxy_helper.rb +1 -1
  49. data/spec/test_helper.rb +27 -3
  50. data/spec/unit/atomic_switcher_spec.rb +2 -2
  51. data/spec/unit/chunker_spec.rb +43 -43
  52. data/spec/unit/connection_spec.rb +2 -2
  53. data/spec/unit/entangler_spec.rb +14 -24
  54. data/spec/unit/printer_spec.rb +2 -6
  55. data/spec/unit/sql_helper_spec.rb +2 -2
  56. data/spec/unit/throttler/{slave_lag_spec.rb → replica_lag_spec.rb} +84 -92
  57. data/spec/unit/throttler/threads_running_spec.rb +18 -0
  58. data/spec/unit/throttler_spec.rb +8 -8
  59. metadata +26 -12
  60. data/.travis.yml +0 -21
  61. data/gemfiles/activerecord_5.2.gemfile +0 -9
  62. data/gemfiles/activerecord_5.2.gemfile.lock +0 -65
  63. data/gemfiles/activerecord_6.0.gemfile.lock +0 -67
@@ -1,161 +1,11 @@
1
+ require 'lhm/throttler/replica_lag'
2
+
1
3
  module Lhm
2
4
  module Throttler
3
-
4
- def self.format_hosts(hosts)
5
- formatted_hosts = []
6
- hosts.each do |host|
7
- if host && !host.match(/localhost/) && !host.match(/127.0.0.1/)
8
- formatted_hosts << host.partition(':')[0]
9
- end
10
- end
11
- formatted_hosts
12
- end
13
-
14
- class SlaveLag
15
- include Command
16
-
17
- INITIAL_TIMEOUT = 0.1
18
- DEFAULT_STRIDE = 2_000
19
- DEFAULT_MAX_ALLOWED_LAG = 10
20
-
21
- MAX_TIMEOUT = INITIAL_TIMEOUT * 1024
22
-
23
- attr_accessor :timeout_seconds, :allowed_lag, :stride, :connection
24
-
5
+ class SlaveLag < ReplicaLag
25
6
  def initialize(options = {})
26
- @timeout_seconds = INITIAL_TIMEOUT
27
- @stride = options[:stride] || DEFAULT_STRIDE
28
- @allowed_lag = options[:allowed_lag] || DEFAULT_MAX_ALLOWED_LAG
29
- @slaves = {}
30
- @get_config = options[:current_config]
31
- @check_only = options[:check_only]
32
- end
33
-
34
- def execute
35
- sleep(throttle_seconds)
36
- end
37
-
38
- private
39
-
40
- def throttle_seconds
41
- lag = max_current_slave_lag
42
-
43
- if lag > @allowed_lag && @timeout_seconds < MAX_TIMEOUT
44
- Lhm.logger.info("Increasing timeout between strides from #{@timeout_seconds} to #{@timeout_seconds * 2} because #{lag} seconds of slave lag detected is greater than the maximum of #{@allowed_lag} seconds allowed.")
45
- @timeout_seconds = @timeout_seconds * 2
46
- elsif lag <= @allowed_lag && @timeout_seconds > INITIAL_TIMEOUT
47
- Lhm.logger.info("Decreasing timeout between strides from #{@timeout_seconds} to #{@timeout_seconds / 2} because #{lag} seconds of slave lag detected is less than or equal to the #{@allowed_lag} seconds allowed.")
48
- @timeout_seconds = @timeout_seconds / 2
49
- else
50
- @timeout_seconds
51
- end
52
- end
53
-
54
- def slaves
55
- @slaves[@connection] ||= get_slaves
56
- end
57
-
58
- def get_slaves
59
- slaves = []
60
- if @check_only.nil? or !@check_only.respond_to?(:call)
61
- slave_hosts = master_slave_hosts
62
- while slave_hosts.any? do
63
- host = slave_hosts.pop
64
- slave = Slave.new(host, @get_config)
65
- if !slaves.map(&:host).include?(host) && slave.connection
66
- slaves << slave
67
- slave_hosts.concat(slave.slave_hosts)
68
- end
69
- end
70
- else
71
- slave_config = @check_only.call
72
- slaves << Slave.new(slave_config['host'], @get_config)
73
- end
74
- slaves
75
- end
76
-
77
- def master_slave_hosts
78
- Throttler.format_hosts(@connection.select_values(Slave::SQL_SELECT_SLAVE_HOSTS))
79
- end
80
-
81
- def max_current_slave_lag
82
- max = slaves.map { |slave| slave.lag }.push(0).max
83
- Lhm.logger.info "Max current slave lag: #{max}"
84
- max
85
- end
86
- end
87
-
88
- class Slave
89
- SQL_SELECT_SLAVE_HOSTS = "SELECT host FROM information_schema.processlist WHERE command LIKE 'Binlog Dump%'"
90
- SQL_SELECT_MAX_SLAVE_LAG = 'SHOW SLAVE STATUS'
91
-
92
- attr_reader :host, :connection
93
-
94
- def initialize(host, connection_config = nil)
95
- @host = host
96
- @connection_config = prepare_connection_config(connection_config)
97
- @connection = client(@connection_config)
98
- end
99
-
100
- def slave_hosts
101
- Throttler.format_hosts(query_connection(SQL_SELECT_SLAVE_HOSTS, 'host'))
102
- end
103
-
104
- def lag
105
- query_connection(SQL_SELECT_MAX_SLAVE_LAG, 'Seconds_Behind_Master').first.to_i
106
- end
107
-
108
- private
109
-
110
- def client(config)
111
- begin
112
- Lhm.logger.info "Connecting to #{@host} on database: #{config[:database]}"
113
- Mysql2::Client.new(config)
114
- rescue Mysql2::Error => e
115
- Lhm.logger.info "Error connecting to #{@host}: #{e}"
116
- nil
117
- end
118
- end
119
-
120
- def prepare_connection_config(config_proc)
121
- config = if config_proc
122
- if config_proc.respond_to?(:call) # if we get a proc
123
- config_proc.call
124
- else
125
- raise ArgumentError, "Expected #{config_proc.inspect} to respond to `call`"
126
- end
127
- else
128
- db_config
129
- end
130
- config.deep_symbolize_keys!
131
- config[:host] = @host
132
- config
133
- end
134
-
135
- def query_connection(query, result)
136
- begin
137
- @connection.query(query).map { |row| row[result] }
138
- rescue Mysql2::Error => e
139
- Lhm.logger.info "Unable to connect and/or query #{host}: #{e}"
140
- [nil]
141
- end
142
- end
143
-
144
- private
145
-
146
- def db_config
147
- if ar_supports_db_config?
148
- ActiveRecord::Base.connection_pool.db_config.configuration_hash.dup
149
- else
150
- ActiveRecord::Base.connection_pool.spec.config.dup
151
- end
152
- end
153
-
154
- def ar_supports_db_config?
155
- # https://api.rubyonrails.org/v6.0/classes/ActiveRecord/ConnectionAdapters/ConnectionPool.html <-- has spec
156
- # vs
157
- # https://api.rubyonrails.org/v6.1/classes/ActiveRecord/ConnectionAdapters/ConnectionPool.html <-- has db_config
158
- ActiveRecord::VERSION::MAJOR > 6 || ActiveRecord::VERSION::MAJOR == 6 && ActiveRecord::VERSION::MINOR >= 1
7
+ Lhm.logger.warn("Class `SlaveLag` is deprecated. Use `ReplicaLag` class instead.")
8
+ super(options)
159
9
  end
160
10
  end
161
11
  end
@@ -3,14 +3,16 @@ module Lhm
3
3
  class ThreadsRunning
4
4
  include Command
5
5
 
6
+ DEFAULT_STRIDE = 2_000
6
7
  DEFAULT_INITIAL_TIMEOUT = 0.1
7
8
  DEFAULT_HEALTHY_RANGE = (0..50)
8
9
 
9
- attr_accessor :timeout_seconds, :healthy_range, :connection
10
+ attr_accessor :timeout_seconds, :healthy_range, :connection, :stride
10
11
  attr_reader :max_timeout_seconds, :initial_timeout_seconds
11
12
 
12
13
  def initialize(options = {})
13
14
  @initial_timeout_seconds = options[:initial_timeout] || DEFAULT_INITIAL_TIMEOUT
15
+ @stride = options[:stride] || DEFAULT_STRIDE
14
16
  @max_timeout_seconds = options[:max_timeout] || (@initial_timeout_seconds * 1024)
15
17
  @timeout_seconds = @initial_timeout_seconds
16
18
  @healthy_range = options[:healthy_range] || DEFAULT_HEALTHY_RANGE
data/lib/lhm/throttler.rb CHANGED
@@ -1,12 +1,16 @@
1
1
  require 'lhm/throttler/time'
2
+ require 'lhm/throttler/replica_lag'
2
3
  require 'lhm/throttler/slave_lag'
3
4
  require 'lhm/throttler/threads_running'
4
5
 
5
6
  module Lhm
6
7
  module Throttler
7
- CLASSES = { :time_throttler => Throttler::Time,
8
- :slave_lag_throttler => Throttler::SlaveLag,
9
- :threads_running_throttler => Throttler::ThreadsRunning }
8
+ CLASSES = {
9
+ :time_throttler => Throttler::Time,
10
+ :replica_lag_throttler => Throttler::ReplicaLag,
11
+ :slave_lag_throttler => Throttler::SlaveLag,
12
+ :threads_running_throttler => Throttler::ThreadsRunning
13
+ }
10
14
 
11
15
  def throttler
12
16
  @throttler ||= Throttler::Time.new
data/lib/lhm/version.rb CHANGED
@@ -2,5 +2,5 @@
2
2
  # Schmidt
3
3
 
4
4
  module Lhm
5
- VERSION = '3.5.5'
5
+ VERSION = '4.1.0'
6
6
  end
@@ -1,19 +1,19 @@
1
1
  #!/bin/bash
2
2
  # Wait for writer
3
3
  echo "Waiting for MySQL-1: "
4
- while ! (mysqladmin ping --host="127.0.0.1" --port=33006 --user=root --password=password --silent 2> /dev/null); do
4
+ while ! (mysqladmin ping --host="127.0.0.1" --port=13006 --user=root --password=password --silent 2> /dev/null); do
5
5
  echo -ne "."
6
6
  sleep 1
7
7
  done
8
8
  # Wait for reader
9
9
  echo "Waiting for MySQL-2: "
10
- while ! (mysqladmin ping --host="127.0.0.1" --port=33007 --user=root --password=password --silent 2> /dev/null); do
10
+ while ! (mysqladmin ping --host="127.0.0.1" --port=13007 --user=root --password=password --silent 2> /dev/null); do
11
11
  echo -ne "."
12
12
  sleep 1
13
13
  done
14
14
  # Wait for proxysql
15
15
  echo "Waiting for ProxySQL:"
16
- while ! (mysqladmin ping --host="127.0.0.1" --port=33005 --user=root --password=password --silent 2> /dev/null); do
16
+ while ! (mysqladmin ping --host="127.0.0.1" --port=13005 --user=root --password=password --silent 2> /dev/null); do
17
17
  echo -ne "."
18
18
  sleep 1
19
19
  done
@@ -3,4 +3,4 @@ CREATE USER IF NOT EXISTS 'writer'@'%' IDENTIFIED BY 'password';
3
3
  CREATE USER IF NOT EXISTS 'reader'@'%' IDENTIFIED BY 'password';
4
4
 
5
5
  CREATE USER IF NOT EXISTS 'replication'@'%' IDENTIFIED BY 'password';
6
- GRANT REPLICATION SLAVE ON *.* TO' replication'@'%' IDENTIFIED BY 'password';
6
+ GRANT REPLICATION SLAVE ON *.* TO' replication'@'%';
data/spec/.lhm.example CHANGED
@@ -1,4 +1,4 @@
1
1
  mysqldir=/usr/local/mysql
2
2
  basedir=~/lhm-cluster
3
3
  master_port=3306
4
- slave_port=3307
4
+ replica_port=3307
data/spec/README.md CHANGED
@@ -1,4 +1,4 @@
1
- # Preparing for master slave integration tests
1
+ # Preparing for master replica integration tests
2
2
 
3
3
  ## Configuration
4
4
 
@@ -7,10 +7,10 @@ create ~/.lhm:
7
7
  mysqldir=/usr/local/mysql
8
8
  basedir=~/lhm-cluster
9
9
  master_port=3306
10
- slave_port=3307
10
+ replica_port=3307
11
11
 
12
12
  mysqldir specifies the location of your mysql install. basedir is the
13
- directory master and slave databases will get installed into.
13
+ directory master and replica databases will get installed into.
14
14
 
15
15
  ## Automatic setup
16
16
 
@@ -18,8 +18,8 @@ directory master and slave databases will get installed into.
18
18
 
19
19
  bin/lhm-spec-clobber.sh
20
20
 
21
- You can set the integration specs up to run against a master slave setup by
22
- running the included that. This deletes the configured lhm master slave setup and reinstalls and configures a master slave setup.
21
+ You can set the integration specs up to run against a master replica setup by
22
+ running the included that. This deletes the configured lhm master replica setup and reinstalls and configures a master replica setup.
23
23
 
24
24
  Follow the manual instructions if you want more control over this process.
25
25
 
@@ -33,7 +33,7 @@ Follow the manual instructions if you want more control over this process.
33
33
 
34
34
  basedir=/opt/lhm-luster
35
35
  mysqld --defaults-file="$basedir/master/my.cnf"
36
- mysqld --defaults-file="$basedir/slave/my.cnf"
36
+ mysqld --defaults-file="$basedir/replica/my.cnf"
37
37
 
38
38
  ### run the grants
39
39
 
@@ -46,13 +46,12 @@ Setup the dependency gems
46
46
  export BUNDLE_GEMFILE=gemfiles/ar-4.2_mysql2.gemfile
47
47
  bundle install
48
48
 
49
- To run specs in slave mode, set the MASTER_SLAVE=1 when running tests:
49
+ To run specs in replica mode, set the MASTER_REPLICA=1 when running tests:
50
50
 
51
- MASTER_SLAVE=1 bundle exec rake specs
51
+ MASTER_REPLICA=1 bundle exec rake specs
52
52
 
53
53
  # connecting
54
54
 
55
55
  you can connect by running (with the respective ports):
56
56
 
57
57
  mysql --protocol=TCP -p3307
58
-
@@ -33,8 +33,8 @@ describe Lhm::AtomicSwitcher do
33
33
  ar_connection = mock()
34
34
  ar_connection.stubs(:data_source_exists?).returns(true)
35
35
  ar_connection.stubs(:active?).returns(true)
36
- ar_connection.stubs(:execute).returns([["dummy"]], [["dummy"]], [["dummy"]])
37
- .then
36
+ ar_connection.stubs(:select_value).returns("dummy")
37
+ ar_connection.stubs(:execute)
38
38
  .raises(ActiveRecord::StatementInvalid, 'Lock wait timeout exceeded; try restarting transaction.')
39
39
  .then
40
40
  .returns([["dummy"]]) # Matches initial host -> triggers retry
@@ -62,16 +62,12 @@ describe Lhm::AtomicSwitcher do
62
62
  ar_connection = mock()
63
63
  ar_connection.stubs(:data_source_exists?).returns(true)
64
64
  ar_connection.stubs(:active?).returns(true)
65
- ar_connection.stubs(:execute).returns([["dummy"]], [["dummy"]], [["dummy"]])
66
- .then
65
+ ar_connection.stubs(:select_value).returns("dummy")
66
+ ar_connection.stubs(:execute)
67
67
  .raises(ActiveRecord::StatementInvalid, 'Lock wait timeout exceeded; try restarting transaction.')
68
68
  .then
69
- .returns([["dummy"]]) # triggers retry 1
70
- .then
71
69
  .raises(ActiveRecord::StatementInvalid, 'Lock wait timeout exceeded; try restarting transaction.')
72
70
  .then
73
- .returns([["dummy"]]) # triggers retry 2
74
- .then
75
71
  .raises(ActiveRecord::StatementInvalid, 'Lock wait timeout exceeded; try restarting transaction.') # triggers retry 2
76
72
 
77
73
  connection = Lhm::Connection.new(connection: ar_connection, options: {
@@ -110,7 +106,7 @@ describe Lhm::AtomicSwitcher do
110
106
  switcher = Lhm::AtomicSwitcher.new(@migration, connection)
111
107
  switcher.run
112
108
 
113
- slave do
109
+ replica do
114
110
  value(data_source_exists?(@origin)).must_equal true
115
111
  value(table_read(@migration.archive_name).columns.keys).must_include 'origin'
116
112
  end
@@ -120,7 +116,7 @@ describe Lhm::AtomicSwitcher do
120
116
  switcher = Lhm::AtomicSwitcher.new(@migration, connection)
121
117
  switcher.run
122
118
 
123
- slave do
119
+ replica do
124
120
  value(data_source_exists?(@destination)).must_equal false
125
121
  value(table_read(@origin.name).columns.keys).must_include 'destination'
126
122
  end
@@ -19,10 +19,10 @@ describe Lhm::ChunkInsert do
19
19
  assert_equal 1, @instance.insert_and_return_count_of_rows_created
20
20
  end
21
21
 
22
- it "inserts the record into the slave" do
22
+ it "inserts the record into the replica" do
23
23
  @instance.insert_and_return_count_of_rows_created
24
24
 
25
- slave do
25
+ replica do
26
26
  value(count_all(@destination.name)).must_equal(1)
27
27
  end
28
28
  end
@@ -28,7 +28,7 @@ describe Lhm::Chunker do
28
28
 
29
29
  Lhm::Chunker.new(@migration, connection, {throttler: throttler, printer: printer} ).run
30
30
 
31
- slave do
31
+ replica do
32
32
  value(count_all(@destination.name)).must_equal(1)
33
33
  end
34
34
 
@@ -41,7 +41,7 @@ describe Lhm::Chunker do
41
41
 
42
42
  Lhm::Chunker.new(@migration, connection, {throttler: throttler, printer: printer} ).run
43
43
 
44
- slave do
44
+ replica do
45
45
  value(count_all(@destination.name)).must_equal(2)
46
46
  end
47
47
  end
@@ -57,7 +57,7 @@ describe Lhm::Chunker do
57
57
 
58
58
  Lhm::Chunker.new(migration, connection, {throttler: throttler, printer: printer} ).run
59
59
 
60
- slave do
60
+ replica do
61
61
  value(count_all(destination.name)).must_equal(2)
62
62
  end
63
63
  end
@@ -74,7 +74,8 @@ describe Lhm::Chunker do
74
74
  Lhm::Chunker.new(migration, connection, {raise_on_warnings: true, throttler: throttler, printer: printer} ).run
75
75
  end
76
76
 
77
- assert_match "Unexpected warning found for inserted row: Duplicate entry '1001' for key 'index_custom_primary_key_on_id'", exception.message
77
+ error_key = index_key("custom_primary_key_dest", "index_custom_primary_key_on_id")
78
+ assert_match "Unexpected warning found for inserted row: Duplicate entry '1001' for key '#{error_key}'", exception.message
78
79
  end
79
80
 
80
81
  it 'should copy and warn on unexpected warnings by default' do
@@ -87,8 +88,10 @@ describe Lhm::Chunker do
87
88
 
88
89
  Lhm::Chunker.new(migration, connection, {throttler: throttler, printer: printer} ).run
89
90
 
91
+ error_key = index_key("custom_primary_key_dest", "index_custom_primary_key_on_id")
92
+
90
93
  assert_equal 2, log_messages.length
91
- assert log_messages[1].include?("Unexpected warning found for inserted row: Duplicate entry '1001' for key 'index_custom_primary_key_on_id'"), log_messages
94
+ assert log_messages[1].include?("Unexpected warning found for inserted row: Duplicate entry '1001' for key '#{error_key}'"), log_messages
92
95
  end
93
96
 
94
97
  it 'should log two times for two unexpected warnings' do
@@ -103,9 +106,11 @@ describe Lhm::Chunker do
103
106
 
104
107
  Lhm::Chunker.new(migration, connection, {throttler: throttler, printer: printer} ).run
105
108
 
109
+ error_key = index_key("custom_primary_key_dest", "index_custom_primary_key_on_id")
110
+
106
111
  assert_equal 3, log_messages.length
107
- assert log_messages[1].include?("Unexpected warning found for inserted row: Duplicate entry '1001' for key 'index_custom_primary_key_on_id'"), log_messages
108
- assert log_messages[2].include?("Unexpected warning found for inserted row: Duplicate entry '1002' for key 'index_custom_primary_key_on_id'"), log_messages
112
+ assert log_messages[1].include?("Unexpected warning found for inserted row: Duplicate entry '1001' for key '#{error_key}'"), log_messages
113
+ assert log_messages[2].include?("Unexpected warning found for inserted row: Duplicate entry '1002' for key '#{error_key}'"), log_messages
109
114
  end
110
115
 
111
116
  it 'should copy and warn on unexpected warnings' do
@@ -118,8 +123,10 @@ describe Lhm::Chunker do
118
123
 
119
124
  Lhm::Chunker.new(migration, connection, {raise_on_warnings: false, throttler: throttler, printer: printer} ).run
120
125
 
126
+ error_key = index_key("custom_primary_key_dest", "index_custom_primary_key_on_id")
127
+
121
128
  assert_equal 2, log_messages.length
122
- assert log_messages[1].include?("Unexpected warning found for inserted row: Duplicate entry '1001' for key 'index_custom_primary_key_on_id'"), log_messages
129
+ assert log_messages[1].include?("Unexpected warning found for inserted row: Duplicate entry '1001' for key '#{error_key}'"), log_messages
123
130
  end
124
131
 
125
132
  it 'should create the modified destination, even if the source is empty' do
@@ -127,7 +134,7 @@ describe Lhm::Chunker do
127
134
 
128
135
  Lhm::Chunker.new(@migration, connection, {throttler: throttler, printer: printer} ).run
129
136
 
130
- slave do
137
+ replica do
131
138
  value(count_all(@destination.name)).must_equal(0)
132
139
  end
133
140
 
@@ -136,20 +143,17 @@ describe Lhm::Chunker do
136
143
  it 'should copy 23 rows from origin to destination in one shot, regardless of the value of the id' do
137
144
  23.times { |n| execute("insert into origin set id = '#{ n * n + 23 }'") }
138
145
 
139
- printer = MiniTest::Mock.new
140
- printer.expect(:notify, :return_value, [Integer, Integer])
141
- printer.expect(:end, :return_value, [])
146
+ printer = mock("printer")
147
+ printer.expects(:notify).with(kind_of(Integer), kind_of(Integer))
148
+ printer.expects(:end)
142
149
 
143
150
  Lhm::Chunker.new(
144
151
  @migration, connection, { throttler: throttler, printer: printer }
145
152
  ).run
146
153
 
147
- slave do
154
+ replica do
148
155
  value(count_all(@destination.name)).must_equal(23)
149
156
  end
150
-
151
- printer.verify
152
-
153
157
  end
154
158
 
155
159
  it 'should copy all the records of a table, even if the last chunk starts with the last record of it.' do
@@ -160,42 +164,40 @@ describe Lhm::Chunker do
160
164
  @migration, connection, { throttler: Lhm::Throttler::Time.new(stride: 10), printer: printer }
161
165
  ).run
162
166
 
163
- slave do
167
+ replica do
164
168
  value(count_all(@destination.name)).must_equal(11)
165
169
  end
166
170
 
167
171
  end
168
172
 
169
- it 'should copy 23 rows from origin to destination in one shot with slave lag based throttler, regardless of the value of the id' do
173
+ it 'should copy 23 rows from origin to destination in one shot with replica lag based throttler, regardless of the value of the id' do
170
174
  23.times { |n| execute("insert into origin set id = '#{ 100000 + n * n + 23 }'") }
171
175
 
172
- printer = MiniTest::Mock.new
173
- printer.expect(:notify, :return_value, [Integer, Integer])
174
- printer.expect(:end, :return_value, [])
176
+ printer = mock("printer")
177
+ printer.expects(:notify).with(kind_of(Integer), kind_of(Integer))
178
+ printer.expects(:end)
175
179
 
176
- Lhm::Throttler::Slave.any_instance.stubs(:slave_hosts).returns(['127.0.0.1'])
177
- Lhm::Throttler::SlaveLag.any_instance.stubs(:master_slave_hosts).returns(['127.0.0.1'])
180
+ Lhm::Throttler::Replica.any_instance.stubs(:replica_hosts).returns(['127.0.0.1'])
181
+ Lhm::Throttler::ReplicaLag.any_instance.stubs(:master_replica_hosts).returns(['127.0.0.1'])
178
182
 
179
183
  Lhm::Chunker.new(
180
- @migration, connection, { throttler: Lhm::Throttler::SlaveLag.new(stride: 100), printer: printer }
184
+ @migration, connection, { throttler: Lhm::Throttler::ReplicaLag.new(stride: 100), printer: printer }
181
185
  ).run
182
186
 
183
- slave do
187
+ replica do
184
188
  value(count_all(@destination.name)).must_equal(23)
185
189
  end
186
-
187
- printer.verify
188
190
  end
189
191
 
190
- it 'should throttle work stride based on slave lag' do
192
+ it 'should throttle work stride based on replica lag' do
191
193
  15.times { |n| execute("insert into origin set id = '#{ (n * n) + 1 }'") }
192
194
 
193
195
  printer = mock()
194
196
  printer.expects(:notify).with(instance_of(Integer), instance_of(Integer)).twice
195
197
  printer.expects(:end)
196
198
 
197
- throttler = Lhm::Throttler::SlaveLag.new(stride: 10, allowed_lag: 0)
198
- def throttler.max_current_slave_lag
199
+ throttler = Lhm::Throttler::ReplicaLag.new(stride: 10, allowed_lag: 0)
200
+ def throttler.max_current_replica_lag
199
201
  1
200
202
  end
201
203
 
@@ -203,14 +205,14 @@ describe Lhm::Chunker do
203
205
  @migration, connection, { throttler: throttler, printer: printer }
204
206
  ).run
205
207
 
206
- assert_equal(Lhm::Throttler::SlaveLag::INITIAL_TIMEOUT * 2 * 2, throttler.timeout_seconds)
208
+ assert_equal(Lhm::Throttler::ReplicaLag::INITIAL_TIMEOUT * 2 * 2, throttler.timeout_seconds)
207
209
 
208
- slave do
210
+ replica do
209
211
  value(count_all(@destination.name)).must_equal(15)
210
212
  end
211
213
  end
212
214
 
213
- it 'should detect a single slave with no lag in the default configuration' do
215
+ it 'should detect a single replica with no lag in the default configuration' do
214
216
  15.times { |n| execute("insert into origin set id = '#{ (n * n) + 1 }'") }
215
217
 
216
218
  printer = mock()
@@ -218,16 +220,16 @@ describe Lhm::Chunker do
218
220
  printer.expects(:verify)
219
221
  printer.expects(:end)
220
222
 
221
- Lhm::Throttler::Slave.any_instance.stubs(:slave_hosts).returns(['127.0.0.1'])
222
- Lhm::Throttler::SlaveLag.any_instance.stubs(:master_slave_hosts).returns(['127.0.0.1'])
223
+ Lhm::Throttler::Replica.any_instance.stubs(:replica_hosts).returns(['127.0.0.1'])
224
+ Lhm::Throttler::ReplicaLag.any_instance.stubs(:master_replica_hosts).returns(['127.0.0.1'])
223
225
 
224
- throttler = Lhm::Throttler::SlaveLag.new(stride: 10, allowed_lag: 0)
226
+ throttler = Lhm::Throttler::ReplicaLag.new(stride: 10, allowed_lag: 0)
225
227
 
226
- if master_slave_mode?
227
- def throttler.slave_connection(slave)
228
+ if master_replica_mode?
229
+ def throttler.replica_connection(replica)
228
230
  config = ActiveRecord::Base.connection_pool.db_config.configuration_hash.dup
229
- config[:host] = slave
230
- config[:port] = 33007
231
+ config[:host] = replica
232
+ config[:port] = 13007
231
233
  ActiveRecord::Base.send('mysql2_connection', config)
232
234
  end
233
235
  end
@@ -236,10 +238,10 @@ describe Lhm::Chunker do
236
238
  @migration, connection, { throttler: throttler, printer: printer }
237
239
  ).run
238
240
 
239
- assert_equal(Lhm::Throttler::SlaveLag::INITIAL_TIMEOUT, throttler.timeout_seconds)
240
- assert_equal(0, throttler.send(:max_current_slave_lag))
241
+ assert_equal(Lhm::Throttler::ReplicaLag::INITIAL_TIMEOUT, throttler.timeout_seconds)
242
+ assert_equal(0, throttler.send(:max_current_replica_lag))
241
243
 
242
- slave do
244
+ replica do
243
245
  value(count_all(@destination.name)).must_equal(15)
244
246
  end
245
247
 
@@ -261,9 +263,17 @@ describe Lhm::Chunker do
261
263
 
262
264
  assert_match "Verification failed, aborting early", exception.message
263
265
 
264
- slave do
266
+ replica do
265
267
  value(count_all(@destination.name)).must_equal(0)
266
268
  end
267
269
  end
268
270
  end
271
+
272
+ def index_key(table_name, index_name)
273
+ if mysql_version.start_with?("8")
274
+ "#{table_name}.#{index_name}"
275
+ else
276
+ index_name
277
+ end
278
+ end
269
279
  end
@@ -2,17 +2,17 @@ master:
2
2
  host: mysql-1
3
3
  user: root
4
4
  password: password
5
- port: 33006
6
- slave:
5
+ port: 13006
6
+ replica:
7
7
  host: mysql-2
8
8
  user: root
9
9
  password: password
10
- port: 33007
10
+ port: 13007
11
11
  proxysql:
12
12
  host: proxysql
13
13
  user: root
14
14
  password: password
15
- port: 33005
15
+ port: 13005
16
16
  master_toxic:
17
17
  host: toxiproxy
18
18
  user: root
@@ -27,7 +27,7 @@ describe Lhm::Entangler do
27
27
  execute("insert into origin (common) values ('inserted')")
28
28
  end
29
29
 
30
- slave do
30
+ replica do
31
31
  value(count(:destination, 'common', 'inserted')).must_equal(1)
32
32
  end
33
33
  end
@@ -39,7 +39,7 @@ describe Lhm::Entangler do
39
39
  execute("delete from origin where common = 'inserted'")
40
40
  end
41
41
 
42
- slave do
42
+ replica do
43
43
  value(count(:destination, 'common', 'inserted')).must_equal(0)
44
44
  end
45
45
  end
@@ -50,7 +50,7 @@ describe Lhm::Entangler do
50
50
  execute("update origin set common = 'updated'")
51
51
  end
52
52
 
53
- slave do
53
+ replica do
54
54
  value(count(:destination, 'common', 'updated')).must_equal(1)
55
55
  end
56
56
  end
@@ -60,7 +60,7 @@ describe Lhm::Entangler do
60
60
 
61
61
  execute("insert into origin (common) values ('inserted')")
62
62
 
63
- slave do
63
+ replica do
64
64
  value(count(:destination, 'common', 'inserted')).must_equal(0)
65
65
  end
66
66
  end