lhm-shopify 3.5.5 → 4.1.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.github/workflows/test.yml +20 -18
- data/Appraisals +8 -19
- data/CHANGELOG.md +16 -0
- data/Gemfile.lock +37 -20
- data/README.md +21 -14
- data/dev.yml +12 -8
- data/docker-compose-mysql-5.7.yml +1 -0
- data/docker-compose-mysql-8.0.yml +63 -0
- data/docker-compose.yml +5 -3
- data/gemfiles/activerecord_6.1.gemfile +1 -0
- data/gemfiles/activerecord_6.1.gemfile.lock +23 -13
- data/gemfiles/{activerecord_7.0.0.alpha2.gemfile → activerecord_7.0.gemfile} +2 -1
- data/gemfiles/{activerecord_7.0.0.alpha2.gemfile.lock → activerecord_7.0.gemfile.lock} +29 -19
- data/gemfiles/{activerecord_6.0.gemfile → activerecord_7.1.gemfile} +1 -1
- data/gemfiles/activerecord_7.1.gemfile.lock +83 -0
- data/lhm.gemspec +2 -1
- data/lib/lhm/atomic_switcher.rb +3 -3
- data/lib/lhm/chunker.rb +4 -4
- data/lib/lhm/connection.rb +9 -1
- data/lib/lhm/sql_helper.rb +1 -1
- data/lib/lhm/sql_retry.rb +36 -18
- data/lib/lhm/table.rb +3 -4
- data/lib/lhm/throttler/replica_lag.rb +166 -0
- data/lib/lhm/throttler/slave_lag.rb +5 -155
- data/lib/lhm/throttler/threads_running.rb +3 -1
- data/lib/lhm/throttler.rb +7 -3
- data/lib/lhm/version.rb +1 -1
- data/scripts/helpers/wait-for-dbs.sh +3 -3
- data/scripts/mysql/writer/create_users.sql +1 -1
- data/spec/.lhm.example +1 -1
- data/spec/README.md +8 -9
- data/spec/integration/atomic_switcher_spec.rb +6 -10
- data/spec/integration/chunk_insert_spec.rb +2 -2
- data/spec/integration/chunker_spec.rb +54 -44
- data/spec/integration/database.yml +4 -4
- data/spec/integration/entangler_spec.rb +4 -4
- data/spec/integration/integration_helper.rb +23 -15
- data/spec/integration/lhm_spec.rb +70 -44
- data/spec/integration/locked_switcher_spec.rb +2 -2
- data/spec/integration/proxysql_spec.rb +10 -10
- data/spec/integration/sql_retry/db_connection_helper.rb +2 -4
- data/spec/integration/sql_retry/lock_wait_spec.rb +7 -8
- data/spec/integration/sql_retry/lock_wait_timeout_test_helper.rb +18 -10
- data/spec/integration/sql_retry/proxysql_helper.rb +1 -1
- data/spec/integration/sql_retry/retry_with_proxysql_spec.rb +1 -2
- data/spec/integration/table_spec.rb +1 -1
- data/spec/integration/toxiproxy_helper.rb +1 -1
- data/spec/test_helper.rb +27 -3
- data/spec/unit/atomic_switcher_spec.rb +2 -2
- data/spec/unit/chunker_spec.rb +43 -43
- data/spec/unit/connection_spec.rb +2 -2
- data/spec/unit/entangler_spec.rb +14 -24
- data/spec/unit/printer_spec.rb +2 -6
- data/spec/unit/sql_helper_spec.rb +2 -2
- data/spec/unit/throttler/{slave_lag_spec.rb → replica_lag_spec.rb} +84 -92
- data/spec/unit/throttler/threads_running_spec.rb +18 -0
- data/spec/unit/throttler_spec.rb +8 -8
- metadata +26 -12
- data/.travis.yml +0 -21
- data/gemfiles/activerecord_5.2.gemfile +0 -9
- data/gemfiles/activerecord_5.2.gemfile.lock +0 -65
- data/gemfiles/activerecord_6.0.gemfile.lock +0 -67
@@ -1,161 +1,11 @@
|
|
1
|
+
require 'lhm/throttler/replica_lag'
|
2
|
+
|
1
3
|
module Lhm
|
2
4
|
module Throttler
|
3
|
-
|
4
|
-
def self.format_hosts(hosts)
|
5
|
-
formatted_hosts = []
|
6
|
-
hosts.each do |host|
|
7
|
-
if host && !host.match(/localhost/) && !host.match(/127.0.0.1/)
|
8
|
-
formatted_hosts << host.partition(':')[0]
|
9
|
-
end
|
10
|
-
end
|
11
|
-
formatted_hosts
|
12
|
-
end
|
13
|
-
|
14
|
-
class SlaveLag
|
15
|
-
include Command
|
16
|
-
|
17
|
-
INITIAL_TIMEOUT = 0.1
|
18
|
-
DEFAULT_STRIDE = 2_000
|
19
|
-
DEFAULT_MAX_ALLOWED_LAG = 10
|
20
|
-
|
21
|
-
MAX_TIMEOUT = INITIAL_TIMEOUT * 1024
|
22
|
-
|
23
|
-
attr_accessor :timeout_seconds, :allowed_lag, :stride, :connection
|
24
|
-
|
5
|
+
class SlaveLag < ReplicaLag
|
25
6
|
def initialize(options = {})
|
26
|
-
|
27
|
-
|
28
|
-
@allowed_lag = options[:allowed_lag] || DEFAULT_MAX_ALLOWED_LAG
|
29
|
-
@slaves = {}
|
30
|
-
@get_config = options[:current_config]
|
31
|
-
@check_only = options[:check_only]
|
32
|
-
end
|
33
|
-
|
34
|
-
def execute
|
35
|
-
sleep(throttle_seconds)
|
36
|
-
end
|
37
|
-
|
38
|
-
private
|
39
|
-
|
40
|
-
def throttle_seconds
|
41
|
-
lag = max_current_slave_lag
|
42
|
-
|
43
|
-
if lag > @allowed_lag && @timeout_seconds < MAX_TIMEOUT
|
44
|
-
Lhm.logger.info("Increasing timeout between strides from #{@timeout_seconds} to #{@timeout_seconds * 2} because #{lag} seconds of slave lag detected is greater than the maximum of #{@allowed_lag} seconds allowed.")
|
45
|
-
@timeout_seconds = @timeout_seconds * 2
|
46
|
-
elsif lag <= @allowed_lag && @timeout_seconds > INITIAL_TIMEOUT
|
47
|
-
Lhm.logger.info("Decreasing timeout between strides from #{@timeout_seconds} to #{@timeout_seconds / 2} because #{lag} seconds of slave lag detected is less than or equal to the #{@allowed_lag} seconds allowed.")
|
48
|
-
@timeout_seconds = @timeout_seconds / 2
|
49
|
-
else
|
50
|
-
@timeout_seconds
|
51
|
-
end
|
52
|
-
end
|
53
|
-
|
54
|
-
def slaves
|
55
|
-
@slaves[@connection] ||= get_slaves
|
56
|
-
end
|
57
|
-
|
58
|
-
def get_slaves
|
59
|
-
slaves = []
|
60
|
-
if @check_only.nil? or !@check_only.respond_to?(:call)
|
61
|
-
slave_hosts = master_slave_hosts
|
62
|
-
while slave_hosts.any? do
|
63
|
-
host = slave_hosts.pop
|
64
|
-
slave = Slave.new(host, @get_config)
|
65
|
-
if !slaves.map(&:host).include?(host) && slave.connection
|
66
|
-
slaves << slave
|
67
|
-
slave_hosts.concat(slave.slave_hosts)
|
68
|
-
end
|
69
|
-
end
|
70
|
-
else
|
71
|
-
slave_config = @check_only.call
|
72
|
-
slaves << Slave.new(slave_config['host'], @get_config)
|
73
|
-
end
|
74
|
-
slaves
|
75
|
-
end
|
76
|
-
|
77
|
-
def master_slave_hosts
|
78
|
-
Throttler.format_hosts(@connection.select_values(Slave::SQL_SELECT_SLAVE_HOSTS))
|
79
|
-
end
|
80
|
-
|
81
|
-
def max_current_slave_lag
|
82
|
-
max = slaves.map { |slave| slave.lag }.push(0).max
|
83
|
-
Lhm.logger.info "Max current slave lag: #{max}"
|
84
|
-
max
|
85
|
-
end
|
86
|
-
end
|
87
|
-
|
88
|
-
class Slave
|
89
|
-
SQL_SELECT_SLAVE_HOSTS = "SELECT host FROM information_schema.processlist WHERE command LIKE 'Binlog Dump%'"
|
90
|
-
SQL_SELECT_MAX_SLAVE_LAG = 'SHOW SLAVE STATUS'
|
91
|
-
|
92
|
-
attr_reader :host, :connection
|
93
|
-
|
94
|
-
def initialize(host, connection_config = nil)
|
95
|
-
@host = host
|
96
|
-
@connection_config = prepare_connection_config(connection_config)
|
97
|
-
@connection = client(@connection_config)
|
98
|
-
end
|
99
|
-
|
100
|
-
def slave_hosts
|
101
|
-
Throttler.format_hosts(query_connection(SQL_SELECT_SLAVE_HOSTS, 'host'))
|
102
|
-
end
|
103
|
-
|
104
|
-
def lag
|
105
|
-
query_connection(SQL_SELECT_MAX_SLAVE_LAG, 'Seconds_Behind_Master').first.to_i
|
106
|
-
end
|
107
|
-
|
108
|
-
private
|
109
|
-
|
110
|
-
def client(config)
|
111
|
-
begin
|
112
|
-
Lhm.logger.info "Connecting to #{@host} on database: #{config[:database]}"
|
113
|
-
Mysql2::Client.new(config)
|
114
|
-
rescue Mysql2::Error => e
|
115
|
-
Lhm.logger.info "Error connecting to #{@host}: #{e}"
|
116
|
-
nil
|
117
|
-
end
|
118
|
-
end
|
119
|
-
|
120
|
-
def prepare_connection_config(config_proc)
|
121
|
-
config = if config_proc
|
122
|
-
if config_proc.respond_to?(:call) # if we get a proc
|
123
|
-
config_proc.call
|
124
|
-
else
|
125
|
-
raise ArgumentError, "Expected #{config_proc.inspect} to respond to `call`"
|
126
|
-
end
|
127
|
-
else
|
128
|
-
db_config
|
129
|
-
end
|
130
|
-
config.deep_symbolize_keys!
|
131
|
-
config[:host] = @host
|
132
|
-
config
|
133
|
-
end
|
134
|
-
|
135
|
-
def query_connection(query, result)
|
136
|
-
begin
|
137
|
-
@connection.query(query).map { |row| row[result] }
|
138
|
-
rescue Mysql2::Error => e
|
139
|
-
Lhm.logger.info "Unable to connect and/or query #{host}: #{e}"
|
140
|
-
[nil]
|
141
|
-
end
|
142
|
-
end
|
143
|
-
|
144
|
-
private
|
145
|
-
|
146
|
-
def db_config
|
147
|
-
if ar_supports_db_config?
|
148
|
-
ActiveRecord::Base.connection_pool.db_config.configuration_hash.dup
|
149
|
-
else
|
150
|
-
ActiveRecord::Base.connection_pool.spec.config.dup
|
151
|
-
end
|
152
|
-
end
|
153
|
-
|
154
|
-
def ar_supports_db_config?
|
155
|
-
# https://api.rubyonrails.org/v6.0/classes/ActiveRecord/ConnectionAdapters/ConnectionPool.html <-- has spec
|
156
|
-
# vs
|
157
|
-
# https://api.rubyonrails.org/v6.1/classes/ActiveRecord/ConnectionAdapters/ConnectionPool.html <-- has db_config
|
158
|
-
ActiveRecord::VERSION::MAJOR > 6 || ActiveRecord::VERSION::MAJOR == 6 && ActiveRecord::VERSION::MINOR >= 1
|
7
|
+
Lhm.logger.warn("Class `SlaveLag` is deprecated. Use `ReplicaLag` class instead.")
|
8
|
+
super(options)
|
159
9
|
end
|
160
10
|
end
|
161
11
|
end
|
@@ -3,14 +3,16 @@ module Lhm
|
|
3
3
|
class ThreadsRunning
|
4
4
|
include Command
|
5
5
|
|
6
|
+
DEFAULT_STRIDE = 2_000
|
6
7
|
DEFAULT_INITIAL_TIMEOUT = 0.1
|
7
8
|
DEFAULT_HEALTHY_RANGE = (0..50)
|
8
9
|
|
9
|
-
attr_accessor :timeout_seconds, :healthy_range, :connection
|
10
|
+
attr_accessor :timeout_seconds, :healthy_range, :connection, :stride
|
10
11
|
attr_reader :max_timeout_seconds, :initial_timeout_seconds
|
11
12
|
|
12
13
|
def initialize(options = {})
|
13
14
|
@initial_timeout_seconds = options[:initial_timeout] || DEFAULT_INITIAL_TIMEOUT
|
15
|
+
@stride = options[:stride] || DEFAULT_STRIDE
|
14
16
|
@max_timeout_seconds = options[:max_timeout] || (@initial_timeout_seconds * 1024)
|
15
17
|
@timeout_seconds = @initial_timeout_seconds
|
16
18
|
@healthy_range = options[:healthy_range] || DEFAULT_HEALTHY_RANGE
|
data/lib/lhm/throttler.rb
CHANGED
@@ -1,12 +1,16 @@
|
|
1
1
|
require 'lhm/throttler/time'
|
2
|
+
require 'lhm/throttler/replica_lag'
|
2
3
|
require 'lhm/throttler/slave_lag'
|
3
4
|
require 'lhm/throttler/threads_running'
|
4
5
|
|
5
6
|
module Lhm
|
6
7
|
module Throttler
|
7
|
-
CLASSES = {
|
8
|
-
|
9
|
-
|
8
|
+
CLASSES = {
|
9
|
+
:time_throttler => Throttler::Time,
|
10
|
+
:replica_lag_throttler => Throttler::ReplicaLag,
|
11
|
+
:slave_lag_throttler => Throttler::SlaveLag,
|
12
|
+
:threads_running_throttler => Throttler::ThreadsRunning
|
13
|
+
}
|
10
14
|
|
11
15
|
def throttler
|
12
16
|
@throttler ||= Throttler::Time.new
|
data/lib/lhm/version.rb
CHANGED
@@ -1,19 +1,19 @@
|
|
1
1
|
#!/bin/bash
|
2
2
|
# Wait for writer
|
3
3
|
echo "Waiting for MySQL-1: "
|
4
|
-
while ! (mysqladmin ping --host="127.0.0.1" --port=
|
4
|
+
while ! (mysqladmin ping --host="127.0.0.1" --port=13006 --user=root --password=password --silent 2> /dev/null); do
|
5
5
|
echo -ne "."
|
6
6
|
sleep 1
|
7
7
|
done
|
8
8
|
# Wait for reader
|
9
9
|
echo "Waiting for MySQL-2: "
|
10
|
-
while ! (mysqladmin ping --host="127.0.0.1" --port=
|
10
|
+
while ! (mysqladmin ping --host="127.0.0.1" --port=13007 --user=root --password=password --silent 2> /dev/null); do
|
11
11
|
echo -ne "."
|
12
12
|
sleep 1
|
13
13
|
done
|
14
14
|
# Wait for proxysql
|
15
15
|
echo "Waiting for ProxySQL:"
|
16
|
-
while ! (mysqladmin ping --host="127.0.0.1" --port=
|
16
|
+
while ! (mysqladmin ping --host="127.0.0.1" --port=13005 --user=root --password=password --silent 2> /dev/null); do
|
17
17
|
echo -ne "."
|
18
18
|
sleep 1
|
19
19
|
done
|
@@ -3,4 +3,4 @@ CREATE USER IF NOT EXISTS 'writer'@'%' IDENTIFIED BY 'password';
|
|
3
3
|
CREATE USER IF NOT EXISTS 'reader'@'%' IDENTIFIED BY 'password';
|
4
4
|
|
5
5
|
CREATE USER IF NOT EXISTS 'replication'@'%' IDENTIFIED BY 'password';
|
6
|
-
GRANT REPLICATION SLAVE ON *.* TO' replication'@'%'
|
6
|
+
GRANT REPLICATION SLAVE ON *.* TO' replication'@'%';
|
data/spec/.lhm.example
CHANGED
data/spec/README.md
CHANGED
@@ -1,4 +1,4 @@
|
|
1
|
-
# Preparing for master
|
1
|
+
# Preparing for master replica integration tests
|
2
2
|
|
3
3
|
## Configuration
|
4
4
|
|
@@ -7,10 +7,10 @@ create ~/.lhm:
|
|
7
7
|
mysqldir=/usr/local/mysql
|
8
8
|
basedir=~/lhm-cluster
|
9
9
|
master_port=3306
|
10
|
-
|
10
|
+
replica_port=3307
|
11
11
|
|
12
12
|
mysqldir specifies the location of your mysql install. basedir is the
|
13
|
-
directory master and
|
13
|
+
directory master and replica databases will get installed into.
|
14
14
|
|
15
15
|
## Automatic setup
|
16
16
|
|
@@ -18,8 +18,8 @@ directory master and slave databases will get installed into.
|
|
18
18
|
|
19
19
|
bin/lhm-spec-clobber.sh
|
20
20
|
|
21
|
-
You can set the integration specs up to run against a master
|
22
|
-
running the included that. This deletes the configured lhm master
|
21
|
+
You can set the integration specs up to run against a master replica setup by
|
22
|
+
running the included that. This deletes the configured lhm master replica setup and reinstalls and configures a master replica setup.
|
23
23
|
|
24
24
|
Follow the manual instructions if you want more control over this process.
|
25
25
|
|
@@ -33,7 +33,7 @@ Follow the manual instructions if you want more control over this process.
|
|
33
33
|
|
34
34
|
basedir=/opt/lhm-luster
|
35
35
|
mysqld --defaults-file="$basedir/master/my.cnf"
|
36
|
-
mysqld --defaults-file="$basedir/
|
36
|
+
mysqld --defaults-file="$basedir/replica/my.cnf"
|
37
37
|
|
38
38
|
### run the grants
|
39
39
|
|
@@ -46,13 +46,12 @@ Setup the dependency gems
|
|
46
46
|
export BUNDLE_GEMFILE=gemfiles/ar-4.2_mysql2.gemfile
|
47
47
|
bundle install
|
48
48
|
|
49
|
-
To run specs in
|
49
|
+
To run specs in replica mode, set the MASTER_REPLICA=1 when running tests:
|
50
50
|
|
51
|
-
|
51
|
+
MASTER_REPLICA=1 bundle exec rake specs
|
52
52
|
|
53
53
|
# connecting
|
54
54
|
|
55
55
|
you can connect by running (with the respective ports):
|
56
56
|
|
57
57
|
mysql --protocol=TCP -p3307
|
58
|
-
|
@@ -33,8 +33,8 @@ describe Lhm::AtomicSwitcher do
|
|
33
33
|
ar_connection = mock()
|
34
34
|
ar_connection.stubs(:data_source_exists?).returns(true)
|
35
35
|
ar_connection.stubs(:active?).returns(true)
|
36
|
-
ar_connection.stubs(:
|
37
|
-
|
36
|
+
ar_connection.stubs(:select_value).returns("dummy")
|
37
|
+
ar_connection.stubs(:execute)
|
38
38
|
.raises(ActiveRecord::StatementInvalid, 'Lock wait timeout exceeded; try restarting transaction.')
|
39
39
|
.then
|
40
40
|
.returns([["dummy"]]) # Matches initial host -> triggers retry
|
@@ -62,16 +62,12 @@ describe Lhm::AtomicSwitcher do
|
|
62
62
|
ar_connection = mock()
|
63
63
|
ar_connection.stubs(:data_source_exists?).returns(true)
|
64
64
|
ar_connection.stubs(:active?).returns(true)
|
65
|
-
ar_connection.stubs(:
|
66
|
-
|
65
|
+
ar_connection.stubs(:select_value).returns("dummy")
|
66
|
+
ar_connection.stubs(:execute)
|
67
67
|
.raises(ActiveRecord::StatementInvalid, 'Lock wait timeout exceeded; try restarting transaction.')
|
68
68
|
.then
|
69
|
-
.returns([["dummy"]]) # triggers retry 1
|
70
|
-
.then
|
71
69
|
.raises(ActiveRecord::StatementInvalid, 'Lock wait timeout exceeded; try restarting transaction.')
|
72
70
|
.then
|
73
|
-
.returns([["dummy"]]) # triggers retry 2
|
74
|
-
.then
|
75
71
|
.raises(ActiveRecord::StatementInvalid, 'Lock wait timeout exceeded; try restarting transaction.') # triggers retry 2
|
76
72
|
|
77
73
|
connection = Lhm::Connection.new(connection: ar_connection, options: {
|
@@ -110,7 +106,7 @@ describe Lhm::AtomicSwitcher do
|
|
110
106
|
switcher = Lhm::AtomicSwitcher.new(@migration, connection)
|
111
107
|
switcher.run
|
112
108
|
|
113
|
-
|
109
|
+
replica do
|
114
110
|
value(data_source_exists?(@origin)).must_equal true
|
115
111
|
value(table_read(@migration.archive_name).columns.keys).must_include 'origin'
|
116
112
|
end
|
@@ -120,7 +116,7 @@ describe Lhm::AtomicSwitcher do
|
|
120
116
|
switcher = Lhm::AtomicSwitcher.new(@migration, connection)
|
121
117
|
switcher.run
|
122
118
|
|
123
|
-
|
119
|
+
replica do
|
124
120
|
value(data_source_exists?(@destination)).must_equal false
|
125
121
|
value(table_read(@origin.name).columns.keys).must_include 'destination'
|
126
122
|
end
|
@@ -19,10 +19,10 @@ describe Lhm::ChunkInsert do
|
|
19
19
|
assert_equal 1, @instance.insert_and_return_count_of_rows_created
|
20
20
|
end
|
21
21
|
|
22
|
-
it "inserts the record into the
|
22
|
+
it "inserts the record into the replica" do
|
23
23
|
@instance.insert_and_return_count_of_rows_created
|
24
24
|
|
25
|
-
|
25
|
+
replica do
|
26
26
|
value(count_all(@destination.name)).must_equal(1)
|
27
27
|
end
|
28
28
|
end
|
@@ -28,7 +28,7 @@ describe Lhm::Chunker do
|
|
28
28
|
|
29
29
|
Lhm::Chunker.new(@migration, connection, {throttler: throttler, printer: printer} ).run
|
30
30
|
|
31
|
-
|
31
|
+
replica do
|
32
32
|
value(count_all(@destination.name)).must_equal(1)
|
33
33
|
end
|
34
34
|
|
@@ -41,7 +41,7 @@ describe Lhm::Chunker do
|
|
41
41
|
|
42
42
|
Lhm::Chunker.new(@migration, connection, {throttler: throttler, printer: printer} ).run
|
43
43
|
|
44
|
-
|
44
|
+
replica do
|
45
45
|
value(count_all(@destination.name)).must_equal(2)
|
46
46
|
end
|
47
47
|
end
|
@@ -57,7 +57,7 @@ describe Lhm::Chunker do
|
|
57
57
|
|
58
58
|
Lhm::Chunker.new(migration, connection, {throttler: throttler, printer: printer} ).run
|
59
59
|
|
60
|
-
|
60
|
+
replica do
|
61
61
|
value(count_all(destination.name)).must_equal(2)
|
62
62
|
end
|
63
63
|
end
|
@@ -74,7 +74,8 @@ describe Lhm::Chunker do
|
|
74
74
|
Lhm::Chunker.new(migration, connection, {raise_on_warnings: true, throttler: throttler, printer: printer} ).run
|
75
75
|
end
|
76
76
|
|
77
|
-
|
77
|
+
error_key = index_key("custom_primary_key_dest", "index_custom_primary_key_on_id")
|
78
|
+
assert_match "Unexpected warning found for inserted row: Duplicate entry '1001' for key '#{error_key}'", exception.message
|
78
79
|
end
|
79
80
|
|
80
81
|
it 'should copy and warn on unexpected warnings by default' do
|
@@ -87,8 +88,10 @@ describe Lhm::Chunker do
|
|
87
88
|
|
88
89
|
Lhm::Chunker.new(migration, connection, {throttler: throttler, printer: printer} ).run
|
89
90
|
|
91
|
+
error_key = index_key("custom_primary_key_dest", "index_custom_primary_key_on_id")
|
92
|
+
|
90
93
|
assert_equal 2, log_messages.length
|
91
|
-
assert log_messages[1].include?("Unexpected warning found for inserted row: Duplicate entry '1001' for key '
|
94
|
+
assert log_messages[1].include?("Unexpected warning found for inserted row: Duplicate entry '1001' for key '#{error_key}'"), log_messages
|
92
95
|
end
|
93
96
|
|
94
97
|
it 'should log two times for two unexpected warnings' do
|
@@ -103,9 +106,11 @@ describe Lhm::Chunker do
|
|
103
106
|
|
104
107
|
Lhm::Chunker.new(migration, connection, {throttler: throttler, printer: printer} ).run
|
105
108
|
|
109
|
+
error_key = index_key("custom_primary_key_dest", "index_custom_primary_key_on_id")
|
110
|
+
|
106
111
|
assert_equal 3, log_messages.length
|
107
|
-
assert log_messages[1].include?("Unexpected warning found for inserted row: Duplicate entry '1001' for key '
|
108
|
-
assert log_messages[2].include?("Unexpected warning found for inserted row: Duplicate entry '1002' for key '
|
112
|
+
assert log_messages[1].include?("Unexpected warning found for inserted row: Duplicate entry '1001' for key '#{error_key}'"), log_messages
|
113
|
+
assert log_messages[2].include?("Unexpected warning found for inserted row: Duplicate entry '1002' for key '#{error_key}'"), log_messages
|
109
114
|
end
|
110
115
|
|
111
116
|
it 'should copy and warn on unexpected warnings' do
|
@@ -118,8 +123,10 @@ describe Lhm::Chunker do
|
|
118
123
|
|
119
124
|
Lhm::Chunker.new(migration, connection, {raise_on_warnings: false, throttler: throttler, printer: printer} ).run
|
120
125
|
|
126
|
+
error_key = index_key("custom_primary_key_dest", "index_custom_primary_key_on_id")
|
127
|
+
|
121
128
|
assert_equal 2, log_messages.length
|
122
|
-
assert log_messages[1].include?("Unexpected warning found for inserted row: Duplicate entry '1001' for key '
|
129
|
+
assert log_messages[1].include?("Unexpected warning found for inserted row: Duplicate entry '1001' for key '#{error_key}'"), log_messages
|
123
130
|
end
|
124
131
|
|
125
132
|
it 'should create the modified destination, even if the source is empty' do
|
@@ -127,7 +134,7 @@ describe Lhm::Chunker do
|
|
127
134
|
|
128
135
|
Lhm::Chunker.new(@migration, connection, {throttler: throttler, printer: printer} ).run
|
129
136
|
|
130
|
-
|
137
|
+
replica do
|
131
138
|
value(count_all(@destination.name)).must_equal(0)
|
132
139
|
end
|
133
140
|
|
@@ -136,20 +143,17 @@ describe Lhm::Chunker do
|
|
136
143
|
it 'should copy 23 rows from origin to destination in one shot, regardless of the value of the id' do
|
137
144
|
23.times { |n| execute("insert into origin set id = '#{ n * n + 23 }'") }
|
138
145
|
|
139
|
-
printer =
|
140
|
-
printer.
|
141
|
-
printer.
|
146
|
+
printer = mock("printer")
|
147
|
+
printer.expects(:notify).with(kind_of(Integer), kind_of(Integer))
|
148
|
+
printer.expects(:end)
|
142
149
|
|
143
150
|
Lhm::Chunker.new(
|
144
151
|
@migration, connection, { throttler: throttler, printer: printer }
|
145
152
|
).run
|
146
153
|
|
147
|
-
|
154
|
+
replica do
|
148
155
|
value(count_all(@destination.name)).must_equal(23)
|
149
156
|
end
|
150
|
-
|
151
|
-
printer.verify
|
152
|
-
|
153
157
|
end
|
154
158
|
|
155
159
|
it 'should copy all the records of a table, even if the last chunk starts with the last record of it.' do
|
@@ -160,42 +164,40 @@ describe Lhm::Chunker do
|
|
160
164
|
@migration, connection, { throttler: Lhm::Throttler::Time.new(stride: 10), printer: printer }
|
161
165
|
).run
|
162
166
|
|
163
|
-
|
167
|
+
replica do
|
164
168
|
value(count_all(@destination.name)).must_equal(11)
|
165
169
|
end
|
166
170
|
|
167
171
|
end
|
168
172
|
|
169
|
-
it 'should copy 23 rows from origin to destination in one shot with
|
173
|
+
it 'should copy 23 rows from origin to destination in one shot with replica lag based throttler, regardless of the value of the id' do
|
170
174
|
23.times { |n| execute("insert into origin set id = '#{ 100000 + n * n + 23 }'") }
|
171
175
|
|
172
|
-
printer =
|
173
|
-
printer.
|
174
|
-
printer.
|
176
|
+
printer = mock("printer")
|
177
|
+
printer.expects(:notify).with(kind_of(Integer), kind_of(Integer))
|
178
|
+
printer.expects(:end)
|
175
179
|
|
176
|
-
Lhm::Throttler::
|
177
|
-
Lhm::Throttler::
|
180
|
+
Lhm::Throttler::Replica.any_instance.stubs(:replica_hosts).returns(['127.0.0.1'])
|
181
|
+
Lhm::Throttler::ReplicaLag.any_instance.stubs(:master_replica_hosts).returns(['127.0.0.1'])
|
178
182
|
|
179
183
|
Lhm::Chunker.new(
|
180
|
-
@migration, connection, { throttler: Lhm::Throttler::
|
184
|
+
@migration, connection, { throttler: Lhm::Throttler::ReplicaLag.new(stride: 100), printer: printer }
|
181
185
|
).run
|
182
186
|
|
183
|
-
|
187
|
+
replica do
|
184
188
|
value(count_all(@destination.name)).must_equal(23)
|
185
189
|
end
|
186
|
-
|
187
|
-
printer.verify
|
188
190
|
end
|
189
191
|
|
190
|
-
it 'should throttle work stride based on
|
192
|
+
it 'should throttle work stride based on replica lag' do
|
191
193
|
15.times { |n| execute("insert into origin set id = '#{ (n * n) + 1 }'") }
|
192
194
|
|
193
195
|
printer = mock()
|
194
196
|
printer.expects(:notify).with(instance_of(Integer), instance_of(Integer)).twice
|
195
197
|
printer.expects(:end)
|
196
198
|
|
197
|
-
throttler = Lhm::Throttler::
|
198
|
-
def throttler.
|
199
|
+
throttler = Lhm::Throttler::ReplicaLag.new(stride: 10, allowed_lag: 0)
|
200
|
+
def throttler.max_current_replica_lag
|
199
201
|
1
|
200
202
|
end
|
201
203
|
|
@@ -203,14 +205,14 @@ describe Lhm::Chunker do
|
|
203
205
|
@migration, connection, { throttler: throttler, printer: printer }
|
204
206
|
).run
|
205
207
|
|
206
|
-
assert_equal(Lhm::Throttler::
|
208
|
+
assert_equal(Lhm::Throttler::ReplicaLag::INITIAL_TIMEOUT * 2 * 2, throttler.timeout_seconds)
|
207
209
|
|
208
|
-
|
210
|
+
replica do
|
209
211
|
value(count_all(@destination.name)).must_equal(15)
|
210
212
|
end
|
211
213
|
end
|
212
214
|
|
213
|
-
it 'should detect a single
|
215
|
+
it 'should detect a single replica with no lag in the default configuration' do
|
214
216
|
15.times { |n| execute("insert into origin set id = '#{ (n * n) + 1 }'") }
|
215
217
|
|
216
218
|
printer = mock()
|
@@ -218,16 +220,16 @@ describe Lhm::Chunker do
|
|
218
220
|
printer.expects(:verify)
|
219
221
|
printer.expects(:end)
|
220
222
|
|
221
|
-
Lhm::Throttler::
|
222
|
-
Lhm::Throttler::
|
223
|
+
Lhm::Throttler::Replica.any_instance.stubs(:replica_hosts).returns(['127.0.0.1'])
|
224
|
+
Lhm::Throttler::ReplicaLag.any_instance.stubs(:master_replica_hosts).returns(['127.0.0.1'])
|
223
225
|
|
224
|
-
throttler = Lhm::Throttler::
|
226
|
+
throttler = Lhm::Throttler::ReplicaLag.new(stride: 10, allowed_lag: 0)
|
225
227
|
|
226
|
-
if
|
227
|
-
def throttler.
|
228
|
+
if master_replica_mode?
|
229
|
+
def throttler.replica_connection(replica)
|
228
230
|
config = ActiveRecord::Base.connection_pool.db_config.configuration_hash.dup
|
229
|
-
config[:host] =
|
230
|
-
config[:port] =
|
231
|
+
config[:host] = replica
|
232
|
+
config[:port] = 13007
|
231
233
|
ActiveRecord::Base.send('mysql2_connection', config)
|
232
234
|
end
|
233
235
|
end
|
@@ -236,10 +238,10 @@ describe Lhm::Chunker do
|
|
236
238
|
@migration, connection, { throttler: throttler, printer: printer }
|
237
239
|
).run
|
238
240
|
|
239
|
-
assert_equal(Lhm::Throttler::
|
240
|
-
assert_equal(0, throttler.send(:
|
241
|
+
assert_equal(Lhm::Throttler::ReplicaLag::INITIAL_TIMEOUT, throttler.timeout_seconds)
|
242
|
+
assert_equal(0, throttler.send(:max_current_replica_lag))
|
241
243
|
|
242
|
-
|
244
|
+
replica do
|
243
245
|
value(count_all(@destination.name)).must_equal(15)
|
244
246
|
end
|
245
247
|
|
@@ -261,9 +263,17 @@ describe Lhm::Chunker do
|
|
261
263
|
|
262
264
|
assert_match "Verification failed, aborting early", exception.message
|
263
265
|
|
264
|
-
|
266
|
+
replica do
|
265
267
|
value(count_all(@destination.name)).must_equal(0)
|
266
268
|
end
|
267
269
|
end
|
268
270
|
end
|
271
|
+
|
272
|
+
def index_key(table_name, index_name)
|
273
|
+
if mysql_version.start_with?("8")
|
274
|
+
"#{table_name}.#{index_name}"
|
275
|
+
else
|
276
|
+
index_name
|
277
|
+
end
|
278
|
+
end
|
269
279
|
end
|
@@ -2,17 +2,17 @@ master:
|
|
2
2
|
host: mysql-1
|
3
3
|
user: root
|
4
4
|
password: password
|
5
|
-
port:
|
6
|
-
|
5
|
+
port: 13006
|
6
|
+
replica:
|
7
7
|
host: mysql-2
|
8
8
|
user: root
|
9
9
|
password: password
|
10
|
-
port:
|
10
|
+
port: 13007
|
11
11
|
proxysql:
|
12
12
|
host: proxysql
|
13
13
|
user: root
|
14
14
|
password: password
|
15
|
-
port:
|
15
|
+
port: 13005
|
16
16
|
master_toxic:
|
17
17
|
host: toxiproxy
|
18
18
|
user: root
|
@@ -27,7 +27,7 @@ describe Lhm::Entangler do
|
|
27
27
|
execute("insert into origin (common) values ('inserted')")
|
28
28
|
end
|
29
29
|
|
30
|
-
|
30
|
+
replica do
|
31
31
|
value(count(:destination, 'common', 'inserted')).must_equal(1)
|
32
32
|
end
|
33
33
|
end
|
@@ -39,7 +39,7 @@ describe Lhm::Entangler do
|
|
39
39
|
execute("delete from origin where common = 'inserted'")
|
40
40
|
end
|
41
41
|
|
42
|
-
|
42
|
+
replica do
|
43
43
|
value(count(:destination, 'common', 'inserted')).must_equal(0)
|
44
44
|
end
|
45
45
|
end
|
@@ -50,7 +50,7 @@ describe Lhm::Entangler do
|
|
50
50
|
execute("update origin set common = 'updated'")
|
51
51
|
end
|
52
52
|
|
53
|
-
|
53
|
+
replica do
|
54
54
|
value(count(:destination, 'common', 'updated')).must_equal(1)
|
55
55
|
end
|
56
56
|
end
|
@@ -60,7 +60,7 @@ describe Lhm::Entangler do
|
|
60
60
|
|
61
61
|
execute("insert into origin (common) values ('inserted')")
|
62
62
|
|
63
|
-
|
63
|
+
replica do
|
64
64
|
value(count(:destination, 'common', 'inserted')).must_equal(0)
|
65
65
|
end
|
66
66
|
end
|