lhm-shopify 3.5.3 → 3.5.4
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/CHANGELOG.md +8 -0
- data/Gemfile.lock +1 -1
- data/README.md +5 -3
- data/Rakefile +6 -5
- data/dev.yml +2 -2
- data/gemfiles/activerecord_5.2.gemfile.lock +1 -1
- data/gemfiles/activerecord_6.0.gemfile.lock +1 -1
- data/gemfiles/activerecord_6.1.gemfile.lock +1 -1
- data/gemfiles/activerecord_7.0.0.alpha2.gemfile.lock +1 -1
- data/lib/lhm/atomic_switcher.rb +4 -3
- data/lib/lhm/chunk_insert.rb +4 -1
- data/lib/lhm/chunker.rb +6 -6
- data/lib/lhm/cleanup/current.rb +4 -1
- data/lib/lhm/connection.rb +24 -25
- data/lib/lhm/entangler.rb +5 -4
- data/lib/lhm/invoker.rb +5 -3
- data/lib/lhm/locked_switcher.rb +2 -0
- data/lib/lhm/sql_retry.rb +19 -13
- data/lib/lhm/version.rb +1 -1
- data/lib/lhm.rb +28 -23
- data/spec/integration/atomic_switcher_spec.rb +28 -17
- data/spec/integration/chunker_spec.rb +7 -5
- data/spec/integration/integration_helper.rb +4 -6
- data/spec/integration/lhm_spec.rb +3 -3
- data/spec/unit/chunker_spec.rb +44 -43
- data/spec/unit/connection_spec.rb +36 -11
- data/spec/unit/entangler_spec.rb +31 -9
- data/spec/unit/throttler/slave_lag_spec.rb +1 -1
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 4f676427b10052113b08c4dda8f648bc11234024deab2f1e96de332e0906a110
|
4
|
+
data.tar.gz: dbd91d1b72d713e9d6e18edb46900971ee89c20563d5e93db9eec746887a12d7
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: be5dc9f5d6d2a3ae62b476fa19455bd0f864b989d6ffbff35a93a0afea4e8d9be4492a4b3bfc2bec1029193114187eaf4eda5bb6dfde9d0be34884bde317c1c1
|
7
|
+
data.tar.gz: aa4e866f74da3255586c1597ae63a3792606bd092e46c3a90335d22b9f6c9b243bc133d99e50233c884b03064dfaeee75983597f84733484e05ec456833f3395
|
data/CHANGELOG.md
CHANGED
@@ -1,3 +1,11 @@
|
|
1
|
+
# 3.5.4 (Dec, 2021)
|
2
|
+
* Refactored the way options are handled internally. Code is now much clearer to understand
|
3
|
+
* Removed optional connection_options from `Lhm.setup` and `Lhm.connection`
|
4
|
+
* Option `reconnect_with_consistent_host` will now be provided with `options` for `Lhm.change_table`
|
5
|
+
|
6
|
+
# 3.5.3 (Dec, 2021)
|
7
|
+
* Adds ProxySQL comments at the end of query to accommodate for internal tool's requirements
|
8
|
+
|
1
9
|
# 3.5.2 (Dec, 2021)
|
2
10
|
* Fixed error on undefined connection, when calling `Lhm.connection` without calling `Lhm.setup` first
|
3
11
|
* Changed `Lhm.connection.connection` to `lhm.connection.ar_connection` for increased clarity and readability
|
data/Gemfile.lock
CHANGED
data/README.md
CHANGED
@@ -113,7 +113,7 @@ tables must be cleaned up.
|
|
113
113
|
LHM can recover from connection loss. However, when used in conjunction with ProxySQL, there are multiple ways that
|
114
114
|
connection loss could induce data loss (if triggered by a failover). Therefore it will perform additional checks to
|
115
115
|
ensure that the MySQL host stays consistent across the schema migrations if the feature is enabled.
|
116
|
-
This is done by tagging every query with `/*maintenance:lhm*/`, which will be recognized by ProxySQL.
|
116
|
+
This is done by tagging every query with `/*maintenance:lhm*/`, which will be recognized by ProxySQL.
|
117
117
|
However, to get this feature working, a new ProxySQL query rule must be added.
|
118
118
|
```cnf
|
119
119
|
{
|
@@ -145,9 +145,11 @@ forwarded to the right target.
|
|
145
145
|
```
|
146
146
|
|
147
147
|
Once these changes are added to the ProxySQL configuration (either through `.cnf` or dynamically through the admin interface),
|
148
|
-
the feature can be enabled. This is done by adding this flag when
|
148
|
+
the feature can be enabled. This is done by adding this flag when providing options to the migration:
|
149
149
|
```ruby
|
150
|
-
Lhm.
|
150
|
+
Lhm.change_table(..., options: {reconnect_with_consistent_host: true}) do |t|
|
151
|
+
...
|
152
|
+
end
|
151
153
|
```
|
152
154
|
**Note**: This feature is disabled by default
|
153
155
|
|
data/Rakefile
CHANGED
@@ -16,15 +16,16 @@ Rake::TestTask.new('integration') do |t|
|
|
16
16
|
t.libs << 'spec'
|
17
17
|
t.test_files = FileList['spec/integration/**/*_spec.rb']
|
18
18
|
t.verbose = true
|
19
|
-
|
19
|
+
end
|
20
20
|
|
21
21
|
Rake::TestTask.new('dev') do |t|
|
22
22
|
t.libs << 'lib'
|
23
23
|
t.libs << 'spec'
|
24
|
-
|
25
|
-
|
26
|
-
|
27
|
-
|
24
|
+
|
25
|
+
files = FileList.new('spec/test_helper.rb')
|
26
|
+
files.add(ENV["SINGLE_TEST"]) if ENV["SINGLE_TEST"]
|
27
|
+
t.test_files = files
|
28
|
+
|
28
29
|
t.verbose = true
|
29
30
|
end
|
30
31
|
|
data/dev.yml
CHANGED
@@ -5,7 +5,7 @@ up:
|
|
5
5
|
or: [mysql@5.7]
|
6
6
|
conflicts: [shopify/shopify/mysql-client, mysql-connector-c, mysql, mysql-client]
|
7
7
|
- wget
|
8
|
-
- ruby: 2.7.
|
8
|
+
- ruby: 2.7.5
|
9
9
|
- bundler
|
10
10
|
- custom:
|
11
11
|
name: Get Appraisal gems
|
@@ -31,7 +31,7 @@ commands:
|
|
31
31
|
if [[ $# -eq 0 ]]; then
|
32
32
|
bundle exec rake unit && bundle exec rake integration
|
33
33
|
else
|
34
|
-
bundle exec rake dev
|
34
|
+
SINGLE_TEST="$@" bundle exec rake dev
|
35
35
|
fi
|
36
36
|
appraisals: bundle exec appraisal rake specs
|
37
37
|
cov: rm -rf coverage; COV=1 bundle exec rake unit && bundle exec rake integration; open coverage/index.html
|
data/lib/lhm/atomic_switcher.rb
CHANGED
@@ -16,12 +16,13 @@ module Lhm
|
|
16
16
|
|
17
17
|
attr_reader :connection
|
18
18
|
|
19
|
-
|
19
|
+
LOG_PREFIX = "AtomicSwitcher"
|
20
|
+
|
21
|
+
def initialize(migration, connection = nil)
|
20
22
|
@migration = migration
|
21
23
|
@connection = connection
|
22
24
|
@origin = migration.origin
|
23
25
|
@destination = migration.destination
|
24
|
-
@retry_options = options[:retriable] || {}
|
25
26
|
end
|
26
27
|
|
27
28
|
def atomic_switch
|
@@ -39,7 +40,7 @@ module Lhm
|
|
39
40
|
private
|
40
41
|
|
41
42
|
def execute
|
42
|
-
@connection.execute(atomic_switch, should_retry: true,
|
43
|
+
@connection.execute(atomic_switch, should_retry: true, log_prefix: LOG_PREFIX)
|
43
44
|
end
|
44
45
|
end
|
45
46
|
end
|
data/lib/lhm/chunk_insert.rb
CHANGED
@@ -3,6 +3,9 @@ require 'lhm/proxysql_helper'
|
|
3
3
|
|
4
4
|
module Lhm
|
5
5
|
class ChunkInsert
|
6
|
+
|
7
|
+
LOG_PREFIX = "ChunkInsert"
|
8
|
+
|
6
9
|
def initialize(migration, connection, lowest, highest, retry_options = {})
|
7
10
|
@migration = migration
|
8
11
|
@connection = connection
|
@@ -12,7 +15,7 @@ module Lhm
|
|
12
15
|
end
|
13
16
|
|
14
17
|
def insert_and_return_count_of_rows_created
|
15
|
-
@connection.update(sql, should_retry: true,
|
18
|
+
@connection.update(sql, should_retry: true, log_prefix: LOG_PREFIX)
|
16
19
|
end
|
17
20
|
|
18
21
|
def sql
|
data/lib/lhm/chunker.rb
CHANGED
@@ -13,6 +13,8 @@ module Lhm
|
|
13
13
|
|
14
14
|
attr_reader :connection
|
15
15
|
|
16
|
+
LOG_PREFIX = "Chunker"
|
17
|
+
|
16
18
|
# Copy from origin to destination in chunks of size `stride`.
|
17
19
|
# Use the `throttler` class to sleep between each stride.
|
18
20
|
def initialize(migration, connection = nil, options = {})
|
@@ -31,9 +33,7 @@ module Lhm
|
|
31
33
|
@retry_options = options[:retriable] || {}
|
32
34
|
@retry_helper = SqlRetry.new(
|
33
35
|
@connection,
|
34
|
-
retry_options:
|
35
|
-
log_prefix: "Chunker"
|
36
|
-
}.merge!(@retry_options)
|
36
|
+
retry_options: @retry_options
|
37
37
|
)
|
38
38
|
end
|
39
39
|
|
@@ -79,7 +79,7 @@ module Lhm
|
|
79
79
|
private
|
80
80
|
|
81
81
|
def raise_on_non_pk_duplicate_warning
|
82
|
-
@connection.execute("show warnings", should_retry: true,
|
82
|
+
@connection.execute("show warnings", should_retry: true, log_prefix: LOG_PREFIX).each do |level, code, message|
|
83
83
|
unless message.match?(/Duplicate entry .+ for key 'PRIMARY'/)
|
84
84
|
m = "Unexpected warning found for inserted row: #{message}"
|
85
85
|
Lhm.logger.warn(m)
|
@@ -94,14 +94,14 @@ module Lhm
|
|
94
94
|
|
95
95
|
def verify_can_run
|
96
96
|
return unless @verifier
|
97
|
-
@retry_helper.with_retries(
|
97
|
+
@retry_helper.with_retries(log_prefix: LOG_PREFIX) do |retriable_connection|
|
98
98
|
raise "Verification failed, aborting early" if !@verifier.call(retriable_connection)
|
99
99
|
end
|
100
100
|
end
|
101
101
|
|
102
102
|
def upper_id(next_id, stride)
|
103
103
|
sql = "select id from `#{ @migration.origin_name }` where id >= #{ next_id } order by id limit 1 offset #{ stride - 1}"
|
104
|
-
top = @connection.select_value(sql, should_retry: true,
|
104
|
+
top = @connection.select_value(sql, should_retry: true, log_prefix: LOG_PREFIX)
|
105
105
|
|
106
106
|
[top ? top.to_i : @limit, @limit].min
|
107
107
|
end
|
data/lib/lhm/cleanup/current.rb
CHANGED
@@ -4,6 +4,9 @@ require 'lhm/sql_retry'
|
|
4
4
|
module Lhm
|
5
5
|
module Cleanup
|
6
6
|
class Current
|
7
|
+
|
8
|
+
LOG_PREFIX = "Current"
|
9
|
+
|
7
10
|
def initialize(run, origin_table_name, connection, options={})
|
8
11
|
@run = run
|
9
12
|
@table_name = TableName.new(origin_table_name)
|
@@ -54,7 +57,7 @@ module Lhm
|
|
54
57
|
|
55
58
|
def execute_ddls
|
56
59
|
ddls.each do |ddl|
|
57
|
-
@connection.execute(ddl, should_retry: true,
|
60
|
+
@connection.execute(ddl, should_retry: true, log_prefix: LOG_PREFIX)
|
58
61
|
end
|
59
62
|
Lhm.logger.info("Dropped triggers on #{@lhm_triggers_for_origin.join(', ')}")
|
60
63
|
Lhm.logger.info("Dropped tables #{@lhm_triggers_for_origin.join(', ')}")
|
data/lib/lhm/connection.rb
CHANGED
@@ -1,13 +1,20 @@
|
|
1
1
|
require 'delegate'
|
2
|
+
require 'forwardable'
|
2
3
|
require 'lhm/sql_retry'
|
3
4
|
|
4
5
|
module Lhm
|
5
6
|
# Lhm::Connection inherits from SingleDelegator. It will forward any unknown method calls to the ActiveRecord
|
6
7
|
# connection.
|
7
8
|
class Connection < SimpleDelegator
|
9
|
+
extend Forwardable
|
8
10
|
|
9
|
-
|
10
|
-
|
11
|
+
# Will delegate the following function to @sql_retry object, while leaving them accessible from the Lhm::Connection
|
12
|
+
# object
|
13
|
+
def_delegators :@sql_retry, :reconnect_with_consistent_host, :reconnect_with_consistent_host=, :retry_config=
|
14
|
+
|
15
|
+
alias ar_connection __getobj__
|
16
|
+
|
17
|
+
def initialize(connection:, options: {})
|
11
18
|
@sql_retry = Lhm::SqlRetry.new(
|
12
19
|
connection,
|
13
20
|
retry_options: options[:retriable] || {},
|
@@ -18,11 +25,6 @@ module Lhm
|
|
18
25
|
super(connection)
|
19
26
|
end
|
20
27
|
|
21
|
-
def ar_connection
|
22
|
-
# Get object from the simple delegator
|
23
|
-
__getobj__
|
24
|
-
end
|
25
|
-
|
26
28
|
def ar_connection=(connection)
|
27
29
|
raise Lhm::Error.new("Lhm::Connection requires an active record connection to operate") if connection.nil?
|
28
30
|
|
@@ -31,46 +33,43 @@ module Lhm
|
|
31
33
|
__setobj__(connection)
|
32
34
|
end
|
33
35
|
|
34
|
-
|
35
|
-
|
36
|
-
|
37
|
-
end
|
38
|
-
|
39
|
-
def execute(query, should_retry: false, retry_options: {})
|
36
|
+
# ActiveRecord::Base overridden methods to incorporate custom retry logic
|
37
|
+
# All other methods will be delegated
|
38
|
+
def execute(query, should_retry: false, log_prefix: nil)
|
40
39
|
if should_retry
|
41
|
-
exec_with_retries(:execute, query,
|
40
|
+
exec_with_retries(:execute, query, log_prefix)
|
42
41
|
else
|
43
42
|
exec(:execute, query)
|
44
43
|
end
|
45
44
|
end
|
46
45
|
|
47
|
-
def update(query, should_retry: false,
|
46
|
+
def update(query, should_retry: false, log_prefix: nil)
|
48
47
|
if should_retry
|
49
|
-
exec_with_retries(:update, query,
|
48
|
+
exec_with_retries(:update, query, log_prefix)
|
50
49
|
else
|
51
50
|
exec(:update, query)
|
52
51
|
end
|
53
52
|
end
|
54
53
|
|
55
|
-
def select_value(query, should_retry: false,
|
54
|
+
def select_value(query, should_retry: false, log_prefix: nil)
|
56
55
|
if should_retry
|
57
|
-
exec_with_retries(:select_value, query,
|
56
|
+
exec_with_retries(:select_value, query, log_prefix)
|
58
57
|
else
|
59
58
|
exec(:select_value, query)
|
60
59
|
end
|
61
60
|
end
|
62
61
|
|
63
|
-
def select_values(query, should_retry: false,
|
62
|
+
def select_values(query, should_retry: false, log_prefix: nil)
|
64
63
|
if should_retry
|
65
|
-
exec_with_retries(:select_values, query,
|
64
|
+
exec_with_retries(:select_values, query, log_prefix)
|
66
65
|
else
|
67
66
|
exec(:select_values, query)
|
68
67
|
end
|
69
68
|
end
|
70
69
|
|
71
|
-
def select_one(query, should_retry: false,
|
70
|
+
def select_one(query, should_retry: false, log_prefix: nil)
|
72
71
|
if should_retry
|
73
|
-
exec_with_retries(:select_one, query,
|
72
|
+
exec_with_retries(:select_one, query, log_prefix)
|
74
73
|
else
|
75
74
|
exec(:select_one, query)
|
76
75
|
end
|
@@ -82,9 +81,9 @@ module Lhm
|
|
82
81
|
ar_connection.public_send(method, Lhm::ProxySQLHelper.tagged(sql))
|
83
82
|
end
|
84
83
|
|
85
|
-
def exec_with_retries(method, sql,
|
86
|
-
|
87
|
-
@sql_retry.with_retries(
|
84
|
+
def exec_with_retries(method, sql, log_prefix=nil)
|
85
|
+
effective_log_prefix = log_prefix || file
|
86
|
+
@sql_retry.with_retries(log_prefix: effective_log_prefix) do |conn|
|
88
87
|
conn.public_send(method, Lhm::ProxySQLHelper.tagged(sql))
|
89
88
|
end
|
90
89
|
end
|
data/lib/lhm/entangler.rb
CHANGED
@@ -13,14 +13,15 @@ module Lhm
|
|
13
13
|
|
14
14
|
attr_reader :connection
|
15
15
|
|
16
|
+
LOG_PREFIX = "Entangler"
|
17
|
+
|
16
18
|
# Creates entanglement between two tables. All creates, updates and deletes
|
17
19
|
# to origin will be repeated on the destination table.
|
18
|
-
def initialize(migration, connection = nil
|
20
|
+
def initialize(migration, connection = nil)
|
19
21
|
@intersection = migration.intersection
|
20
22
|
@origin = migration.origin
|
21
23
|
@destination = migration.destination
|
22
24
|
@connection = connection
|
23
|
-
@retry_options = options[:retriable] || {}
|
24
25
|
end
|
25
26
|
|
26
27
|
def entangle
|
@@ -86,14 +87,14 @@ module Lhm
|
|
86
87
|
|
87
88
|
def before
|
88
89
|
entangle.each do |stmt|
|
89
|
-
@connection.execute(stmt, should_retry: true,
|
90
|
+
@connection.execute(stmt, should_retry: true, log_prefix: LOG_PREFIX)
|
90
91
|
end
|
91
92
|
Lhm.logger.info("Created triggers on #{@origin.name}")
|
92
93
|
end
|
93
94
|
|
94
95
|
def after
|
95
96
|
untangle.each do |stmt|
|
96
|
-
@connection.execute(stmt, should_retry: true,
|
97
|
+
@connection.execute(stmt, should_retry: true, log_prefix: LOG_PREFIX)
|
97
98
|
end
|
98
99
|
Lhm.logger.info("Dropped triggers on #{@origin.name}")
|
99
100
|
end
|
data/lib/lhm/invoker.rb
CHANGED
@@ -16,8 +16,8 @@ module Lhm
|
|
16
16
|
class Invoker
|
17
17
|
include SqlHelper
|
18
18
|
LOCK_WAIT_TIMEOUT_DELTA = 10
|
19
|
-
INNODB_LOCK_WAIT_TIMEOUT_MAX=1073741824.freeze # https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_lock_wait_timeout
|
20
|
-
LOCK_WAIT_TIMEOUT_MAX=31536000.freeze # https://dev.mysql.com/doc/refman/5.7/en/server-system-variables.html
|
19
|
+
INNODB_LOCK_WAIT_TIMEOUT_MAX = 1073741824.freeze # https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_lock_wait_timeout
|
20
|
+
LOCK_WAIT_TIMEOUT_MAX = 31536000.freeze # https://dev.mysql.com/doc/refman/5.7/en/server-system-variables.html
|
21
21
|
|
22
22
|
attr_reader :migrator, :connection
|
23
23
|
|
@@ -49,7 +49,7 @@ module Lhm
|
|
49
49
|
normalize_options(options)
|
50
50
|
set_session_lock_wait_timeouts
|
51
51
|
migration = @migrator.run
|
52
|
-
entangler = Entangler.new(migration, @connection
|
52
|
+
entangler = Entangler.new(migration, @connection)
|
53
53
|
|
54
54
|
entangler.run do
|
55
55
|
options[:verifier] ||= Proc.new { |conn| triggers_still_exist?(conn, entangler) }
|
@@ -90,6 +90,8 @@ module Lhm
|
|
90
90
|
options[:throttler] = Lhm.throttler
|
91
91
|
end
|
92
92
|
|
93
|
+
Lhm.connection.retry_config = options[:retriable] || {}
|
94
|
+
|
93
95
|
rescue => e
|
94
96
|
Lhm.logger.error "LHM run failed with exception=#{e.class} message=#{e.message}"
|
95
97
|
raise
|
data/lib/lhm/locked_switcher.rb
CHANGED
data/lib/lhm/sql_retry.rb
CHANGED
@@ -30,21 +30,15 @@ module Lhm
|
|
30
30
|
|
31
31
|
def initialize(connection, retry_options: {}, reconnect_with_consistent_host: false)
|
32
32
|
@connection = connection
|
33
|
-
|
34
|
-
|
35
|
-
if (@reconnect_with_consistent_host = reconnect_with_consistent_host)
|
36
|
-
@initial_hostname = hostname
|
37
|
-
@initial_server_id = server_id
|
38
|
-
end
|
33
|
+
self.retry_config = retry_options
|
34
|
+
self.reconnect_with_consistent_host = reconnect_with_consistent_host
|
39
35
|
end
|
40
36
|
|
41
37
|
# Complete explanation of algorithm: https://github.com/Shopify/lhm/pull/112
|
42
|
-
def with_retries(
|
43
|
-
@log_prefix =
|
38
|
+
def with_retries(log_prefix: nil)
|
39
|
+
@log_prefix = log_prefix || "" # No prefix. Just logs
|
44
40
|
|
45
|
-
|
46
|
-
|
47
|
-
Retriable.retriable(retry_config) do
|
41
|
+
Retriable.retriable(@retry_config) do
|
48
42
|
# Using begin -> rescue -> end for Ruby 2.4 compatibility
|
49
43
|
begin
|
50
44
|
if @reconnect_with_consistent_host
|
@@ -62,8 +56,20 @@ module Lhm
|
|
62
56
|
end
|
63
57
|
end
|
64
58
|
|
65
|
-
|
66
|
-
|
59
|
+
# Both attributes will have defined setters
|
60
|
+
attr_reader :retry_config, :reconnect_with_consistent_host
|
61
|
+
attr_accessor :connection
|
62
|
+
|
63
|
+
def retry_config=(retry_options)
|
64
|
+
@retry_config = default_retry_config.dup.merge!(retry_options)
|
65
|
+
end
|
66
|
+
|
67
|
+
def reconnect_with_consistent_host=(reconnect)
|
68
|
+
if (@reconnect_with_consistent_host = reconnect)
|
69
|
+
@initial_hostname = hostname
|
70
|
+
@initial_server_id = server_id
|
71
|
+
end
|
72
|
+
end
|
67
73
|
|
68
74
|
private
|
69
75
|
|
data/lib/lhm/version.rb
CHANGED
data/lib/lhm.rb
CHANGED
@@ -46,15 +46,19 @@ module Lhm
|
|
46
46
|
# Use atomic switch to rename tables (defaults to: true)
|
47
47
|
# If using a version of mysql affected by atomic switch bug, LHM forces user
|
48
48
|
# to set this option (see SqlHelper#supports_atomic_switch?)
|
49
|
+
# @option options [Boolean] :reconnect_with_consistent_host
|
50
|
+
# Active / Deactivate ProxySQL-aware reconnection procedure (default to: false)
|
49
51
|
# @yield [Migrator] Yielded Migrator object records the changes
|
50
52
|
# @return [Boolean] Returns true if the migration finishes
|
51
53
|
# @raise [Error] Raises Lhm::Error in case of a error and aborts the migration
|
52
54
|
def change_table(table_name, options = {}, &block)
|
53
|
-
|
54
|
-
|
55
|
-
|
56
|
-
|
57
|
-
|
55
|
+
with_flags(options) do
|
56
|
+
origin = Table.parse(table_name, connection)
|
57
|
+
invoker = Invoker.new(origin, connection)
|
58
|
+
block.call(invoker.migrator)
|
59
|
+
invoker.run(options)
|
60
|
+
true
|
61
|
+
end
|
58
62
|
end
|
59
63
|
|
60
64
|
# Cleanup tables and triggers
|
@@ -86,27 +90,16 @@ module Lhm
|
|
86
90
|
# Setups DB connection
|
87
91
|
#
|
88
92
|
# @param [ActiveRecord::Base] connection ActiveRecord Connection
|
89
|
-
|
90
|
-
|
91
|
-
# Active / Deactivate ProxySQL-aware reconnection procedure (default to: false)
|
92
|
-
def setup(connection, connection_options = {})
|
93
|
-
@@connection = Connection.new(connection: connection, options: connection_options)
|
93
|
+
def setup(connection)
|
94
|
+
@@connection = Connection.new(connection: connection)
|
94
95
|
end
|
95
96
|
|
96
|
-
#
|
97
|
-
|
98
|
-
# @param [Hash] connection_options Optional options (defaults to: empty hash)
|
99
|
-
# @option connection_options [Boolean] :reconnect_with_consistent_host
|
100
|
-
# Active / Deactivate ProxySQL-aware reconnection procedure (default to: false)
|
101
|
-
def connection(connection_options = nil)
|
97
|
+
# Returns DB connection (or initializes it if not created yet)
|
98
|
+
def connection
|
102
99
|
@@connection ||= begin
|
103
|
-
|
104
|
-
|
105
|
-
|
106
|
-
|
107
|
-
@@connection.process_connection_options(connection_options) unless connection_options.nil?
|
108
|
-
|
109
|
-
@@connection
|
100
|
+
raise 'Please call Lhm.setup' unless defined?(ActiveRecord)
|
101
|
+
@@connection = Connection.new(connection: ActiveRecord::Base.connection)
|
102
|
+
end
|
110
103
|
end
|
111
104
|
|
112
105
|
def self.logger=(new_logger)
|
@@ -148,4 +141,16 @@ module Lhm
|
|
148
141
|
false
|
149
142
|
end
|
150
143
|
end
|
144
|
+
|
145
|
+
def with_flags(options)
|
146
|
+
old_flags = {
|
147
|
+
reconnect_with_consistent_host: Lhm.connection.reconnect_with_consistent_host,
|
148
|
+
}
|
149
|
+
|
150
|
+
Lhm.connection.reconnect_with_consistent_host = options[:reconnect_with_consistent_host] || false
|
151
|
+
|
152
|
+
yield
|
153
|
+
ensure
|
154
|
+
Lhm.connection.reconnect_with_consistent_host = old_flags[:reconnect_with_consistent_host]
|
155
|
+
end
|
151
156
|
end
|
@@ -39,10 +39,15 @@ describe Lhm::AtomicSwitcher do
|
|
39
39
|
.then
|
40
40
|
.returns([["dummy"]]) # Matches initial host -> triggers retry
|
41
41
|
|
42
|
-
connection = Lhm::Connection.new(connection: ar_connection, options: {
|
42
|
+
connection = Lhm::Connection.new(connection: ar_connection, options: {
|
43
|
+
reconnect_with_consistent_host: true,
|
44
|
+
retriable: {
|
45
|
+
tries: 3,
|
46
|
+
base_interval: 0
|
47
|
+
}
|
48
|
+
})
|
43
49
|
|
44
|
-
|
45
|
-
switcher = Lhm::AtomicSwitcher.new(@migration, connection, retriable: { tries: 3, base_interval: 0 })
|
50
|
+
switcher = Lhm::AtomicSwitcher.new(@migration, connection)
|
46
51
|
|
47
52
|
assert switcher.run
|
48
53
|
|
@@ -58,20 +63,26 @@ describe Lhm::AtomicSwitcher do
|
|
58
63
|
ar_connection.stubs(:data_source_exists?).returns(true)
|
59
64
|
ar_connection.stubs(:active?).returns(true)
|
60
65
|
ar_connection.stubs(:execute).returns([["dummy"]], [["dummy"]], [["dummy"]])
|
61
|
-
|
62
|
-
|
63
|
-
|
64
|
-
|
65
|
-
|
66
|
-
|
67
|
-
|
68
|
-
|
69
|
-
|
70
|
-
|
71
|
-
|
72
|
-
connection = Lhm::Connection.new(connection: ar_connection, options: {
|
73
|
-
|
74
|
-
|
66
|
+
.then
|
67
|
+
.raises(ActiveRecord::StatementInvalid, 'Lock wait timeout exceeded; try restarting transaction.')
|
68
|
+
.then
|
69
|
+
.returns([["dummy"]]) # triggers retry 1
|
70
|
+
.then
|
71
|
+
.raises(ActiveRecord::StatementInvalid, 'Lock wait timeout exceeded; try restarting transaction.')
|
72
|
+
.then
|
73
|
+
.returns([["dummy"]]) # triggers retry 2
|
74
|
+
.then
|
75
|
+
.raises(ActiveRecord::StatementInvalid, 'Lock wait timeout exceeded; try restarting transaction.') # triggers retry 2
|
76
|
+
|
77
|
+
connection = Lhm::Connection.new(connection: ar_connection, options: {
|
78
|
+
reconnect_with_consistent_host: true,
|
79
|
+
retriable: {
|
80
|
+
tries: 2,
|
81
|
+
base_interval: 0
|
82
|
+
}
|
83
|
+
})
|
84
|
+
|
85
|
+
switcher = Lhm::AtomicSwitcher.new(@migration, connection)
|
75
86
|
|
76
87
|
assert_raises(ActiveRecord::StatementInvalid) { switcher.run }
|
77
88
|
end
|
@@ -173,6 +173,9 @@ describe Lhm::Chunker do
|
|
173
173
|
printer.expect(:notify, :return_value, [Integer, Integer])
|
174
174
|
printer.expect(:end, :return_value, [])
|
175
175
|
|
176
|
+
Lhm::Throttler::Slave.any_instance.stubs(:slave_hosts).returns(['127.0.0.1'])
|
177
|
+
Lhm::Throttler::SlaveLag.any_instance.stubs(:master_slave_hosts).returns(['127.0.0.1'])
|
178
|
+
|
176
179
|
Lhm::Chunker.new(
|
177
180
|
@migration, connection, { throttler: Lhm::Throttler::SlaveLag.new(stride: 100), printer: printer }
|
178
181
|
).run
|
@@ -215,17 +218,16 @@ describe Lhm::Chunker do
|
|
215
218
|
printer.expects(:verify)
|
216
219
|
printer.expects(:end)
|
217
220
|
|
218
|
-
|
221
|
+
Lhm::Throttler::Slave.any_instance.stubs(:slave_hosts).returns(['127.0.0.1'])
|
222
|
+
Lhm::Throttler::SlaveLag.any_instance.stubs(:master_slave_hosts).returns(['127.0.0.1'])
|
219
223
|
|
220
|
-
|
221
|
-
['127.0.0.1']
|
222
|
-
end
|
224
|
+
throttler = Lhm::Throttler::SlaveLag.new(stride: 10, allowed_lag: 0)
|
223
225
|
|
224
226
|
if master_slave_mode?
|
225
227
|
def throttler.slave_connection(slave)
|
226
228
|
config = ActiveRecord::Base.connection_pool.db_config.configuration_hash.dup
|
227
229
|
config[:host] = slave
|
228
|
-
config[:port] =
|
230
|
+
config[:port] = 33007
|
229
231
|
ActiveRecord::Base.send('mysql2_connection', config)
|
230
232
|
end
|
231
233
|
end
|
@@ -62,18 +62,16 @@ module IntegrationHelper
|
|
62
62
|
)
|
63
63
|
end
|
64
64
|
|
65
|
-
def connect_master_with_toxiproxy!
|
65
|
+
def connect_master_with_toxiproxy!
|
66
66
|
connect!(
|
67
67
|
'127.0.0.1',
|
68
68
|
$db_config['master_toxic']['port'],
|
69
69
|
$db_config['master_toxic']['user'],
|
70
|
-
$db_config['master_toxic']['password']
|
71
|
-
with_retry)
|
70
|
+
$db_config['master_toxic']['password'])
|
72
71
|
end
|
73
72
|
|
74
|
-
def connect!(hostname, port, user, password
|
75
|
-
|
76
|
-
Lhm.setup(adapter,{reconnect_with_consistent_host: with_retry})
|
73
|
+
def connect!(hostname, port, user, password)
|
74
|
+
Lhm.setup(ar_conn(hostname, port, user, password))
|
77
75
|
unless defined?(@@cleaned_up)
|
78
76
|
Lhm.cleanup(true)
|
79
77
|
@@cleaned_up = true
|
@@ -592,7 +592,7 @@ describe Lhm do
|
|
592
592
|
end
|
593
593
|
|
594
594
|
it " should not try to reconnect if reconnect_with_consistent_host is not provided" do
|
595
|
-
connect_master_with_toxiproxy!
|
595
|
+
connect_master_with_toxiproxy!
|
596
596
|
|
597
597
|
table_create(:users)
|
598
598
|
100.times { |n| execute("insert into users set reference = '#{ n }'") }
|
@@ -610,7 +610,7 @@ describe Lhm do
|
|
610
610
|
end
|
611
611
|
|
612
612
|
it "should reconnect if reconnect_with_consistent_host is true" do
|
613
|
-
connect_master_with_toxiproxy!
|
613
|
+
connect_master_with_toxiproxy!
|
614
614
|
mysql_disabled = false
|
615
615
|
|
616
616
|
table_create(:users)
|
@@ -635,7 +635,7 @@ describe Lhm do
|
|
635
635
|
method_added(:insert_and_return_count_of_rows_created)
|
636
636
|
end
|
637
637
|
|
638
|
-
Lhm.change_table(:users, :
|
638
|
+
Lhm.change_table(:users, atomic_switch: false, reconnect_with_consistent_host: true) do |t|
|
639
639
|
t.ddl("ALTER TABLE #{t.name} CHANGE id id bigint (20) NOT NULL")
|
640
640
|
t.ddl("ALTER TABLE #{t.name} DROP PRIMARY KEY, ADD PRIMARY KEY (username, id)")
|
641
641
|
t.ddl("ALTER TABLE #{t.name} ADD INDEX (id)")
|
data/spec/unit/chunker_spec.rb
CHANGED
@@ -12,7 +12,8 @@ require 'lhm/connection'
|
|
12
12
|
describe Lhm::Chunker do
|
13
13
|
include UnitHelper
|
14
14
|
|
15
|
-
|
15
|
+
EXPECTED_RETRY_FLAGS_CHUNKER = {:should_retry => true, :log_prefix => "Chunker"}
|
16
|
+
EXPECTED_RETRY_FLAGS_CHUNK_INSERT = {:should_retry => true, :log_prefix => "ChunkInsert"}
|
16
17
|
|
17
18
|
before(:each) do
|
18
19
|
@origin = Lhm::Table.new('foo')
|
@@ -41,11 +42,11 @@ describe Lhm::Chunker do
|
|
41
42
|
5
|
42
43
|
end
|
43
44
|
|
44
|
-
@connection.expects(:select_value).with(regexp_matches(/where id >= 1 order by id limit 1 offset 4/),
|
45
|
-
@connection.expects(:select_value).with(regexp_matches(/where id >= 8 order by id limit 1 offset 4/),
|
46
|
-
@connection.expects(:update).with(regexp_matches(/between 1 and 7/),
|
47
|
-
@connection.expects(:update).with(regexp_matches(/between 8 and 10/),
|
48
|
-
@connection.expects(:execute).twice.with(regexp_matches(/show warnings/),
|
45
|
+
@connection.expects(:select_value).with(regexp_matches(/where id >= 1 order by id limit 1 offset 4/),EXPECTED_RETRY_FLAGS_CHUNKER).returns(7)
|
46
|
+
@connection.expects(:select_value).with(regexp_matches(/where id >= 8 order by id limit 1 offset 4/),EXPECTED_RETRY_FLAGS_CHUNKER).returns(21)
|
47
|
+
@connection.expects(:update).with(regexp_matches(/between 1 and 7/),EXPECTED_RETRY_FLAGS_CHUNK_INSERT).returns(2)
|
48
|
+
@connection.expects(:update).with(regexp_matches(/between 8 and 10/),EXPECTED_RETRY_FLAGS_CHUNK_INSERT).returns(2)
|
49
|
+
@connection.expects(:execute).twice.with(regexp_matches(/show warnings/),EXPECTED_RETRY_FLAGS_CHUNKER).returns([])
|
49
50
|
|
50
51
|
@chunker.run
|
51
52
|
end
|
@@ -56,17 +57,17 @@ describe Lhm::Chunker do
|
|
56
57
|
2
|
57
58
|
end
|
58
59
|
|
59
|
-
@connection.expects(:select_value).with(regexp_matches(/where id >= 1 order by id limit 1 offset 1/),
|
60
|
-
@connection.expects(:select_value).with(regexp_matches(/where id >= 3 order by id limit 1 offset 1/),
|
61
|
-
@connection.expects(:select_value).with(regexp_matches(/where id >= 5 order by id limit 1 offset 1/),
|
62
|
-
@connection.expects(:select_value).with(regexp_matches(/where id >= 7 order by id limit 1 offset 1/),
|
63
|
-
@connection.expects(:select_value).with(regexp_matches(/where id >= 9 order by id limit 1 offset 1/),
|
60
|
+
@connection.expects(:select_value).with(regexp_matches(/where id >= 1 order by id limit 1 offset 1/),EXPECTED_RETRY_FLAGS_CHUNKER).returns(2)
|
61
|
+
@connection.expects(:select_value).with(regexp_matches(/where id >= 3 order by id limit 1 offset 1/),EXPECTED_RETRY_FLAGS_CHUNKER).returns(4)
|
62
|
+
@connection.expects(:select_value).with(regexp_matches(/where id >= 5 order by id limit 1 offset 1/),EXPECTED_RETRY_FLAGS_CHUNKER).returns(6)
|
63
|
+
@connection.expects(:select_value).with(regexp_matches(/where id >= 7 order by id limit 1 offset 1/),EXPECTED_RETRY_FLAGS_CHUNKER).returns(8)
|
64
|
+
@connection.expects(:select_value).with(regexp_matches(/where id >= 9 order by id limit 1 offset 1/),EXPECTED_RETRY_FLAGS_CHUNKER).returns(10)
|
64
65
|
|
65
|
-
@connection.expects(:update).with(regexp_matches(/between 1 and 2/),
|
66
|
-
@connection.expects(:update).with(regexp_matches(/between 3 and 4/),
|
67
|
-
@connection.expects(:update).with(regexp_matches(/between 5 and 6/),
|
68
|
-
@connection.expects(:update).with(regexp_matches(/between 7 and 8/),
|
69
|
-
@connection.expects(:update).with(regexp_matches(/between 9 and 10/),
|
66
|
+
@connection.expects(:update).with(regexp_matches(/between 1 and 2/),EXPECTED_RETRY_FLAGS_CHUNK_INSERT).returns(2)
|
67
|
+
@connection.expects(:update).with(regexp_matches(/between 3 and 4/),EXPECTED_RETRY_FLAGS_CHUNK_INSERT).returns(2)
|
68
|
+
@connection.expects(:update).with(regexp_matches(/between 5 and 6/),EXPECTED_RETRY_FLAGS_CHUNK_INSERT).returns(2)
|
69
|
+
@connection.expects(:update).with(regexp_matches(/between 7 and 8/),EXPECTED_RETRY_FLAGS_CHUNK_INSERT).returns(2)
|
70
|
+
@connection.expects(:update).with(regexp_matches(/between 9 and 10/),EXPECTED_RETRY_FLAGS_CHUNK_INSERT).returns(2)
|
70
71
|
|
71
72
|
@chunker.run
|
72
73
|
end
|
@@ -83,17 +84,17 @@ describe Lhm::Chunker do
|
|
83
84
|
end
|
84
85
|
end
|
85
86
|
|
86
|
-
@connection.expects(:select_value).with(regexp_matches(/where id >= 1 order by id limit 1 offset 1/),
|
87
|
-
@connection.expects(:select_value).with(regexp_matches(/where id >= 3 order by id limit 1 offset 2/),
|
88
|
-
@connection.expects(:select_value).with(regexp_matches(/where id >= 6 order by id limit 1 offset 2/),
|
89
|
-
@connection.expects(:select_value).with(regexp_matches(/where id >= 9 order by id limit 1 offset 2/),
|
87
|
+
@connection.expects(:select_value).with(regexp_matches(/where id >= 1 order by id limit 1 offset 1/),EXPECTED_RETRY_FLAGS_CHUNKER).returns(2)
|
88
|
+
@connection.expects(:select_value).with(regexp_matches(/where id >= 3 order by id limit 1 offset 2/),EXPECTED_RETRY_FLAGS_CHUNKER).returns(5)
|
89
|
+
@connection.expects(:select_value).with(regexp_matches(/where id >= 6 order by id limit 1 offset 2/),EXPECTED_RETRY_FLAGS_CHUNKER).returns(8)
|
90
|
+
@connection.expects(:select_value).with(regexp_matches(/where id >= 9 order by id limit 1 offset 2/),EXPECTED_RETRY_FLAGS_CHUNKER).returns(nil)
|
90
91
|
|
91
|
-
@connection.expects(:update).with(regexp_matches(/between 1 and 2/),
|
92
|
-
@connection.expects(:update).with(regexp_matches(/between 3 and 5/),
|
93
|
-
@connection.expects(:update).with(regexp_matches(/between 6 and 8/),
|
94
|
-
@connection.expects(:update).with(regexp_matches(/between 9 and 10/),
|
92
|
+
@connection.expects(:update).with(regexp_matches(/between 1 and 2/),EXPECTED_RETRY_FLAGS_CHUNK_INSERT).returns(2)
|
93
|
+
@connection.expects(:update).with(regexp_matches(/between 3 and 5/),EXPECTED_RETRY_FLAGS_CHUNK_INSERT).returns(2)
|
94
|
+
@connection.expects(:update).with(regexp_matches(/between 6 and 8/),EXPECTED_RETRY_FLAGS_CHUNK_INSERT).returns(2)
|
95
|
+
@connection.expects(:update).with(regexp_matches(/between 9 and 10/),EXPECTED_RETRY_FLAGS_CHUNK_INSERT).returns(2)
|
95
96
|
|
96
|
-
@connection.expects(:execute).twice.with(regexp_matches(/show warnings/),
|
97
|
+
@connection.expects(:execute).twice.with(regexp_matches(/show warnings/),EXPECTED_RETRY_FLAGS_CHUNKER).returns([])
|
97
98
|
|
98
99
|
@chunker.run
|
99
100
|
end
|
@@ -103,8 +104,8 @@ describe Lhm::Chunker do
|
|
103
104
|
:start => 1,
|
104
105
|
:limit => 1)
|
105
106
|
|
106
|
-
@connection.expects(:select_value).with(regexp_matches(/where id >= 1 order by id limit 1 offset 0/),
|
107
|
-
@connection.expects(:update).with(regexp_matches(/between 1 and 1/),
|
107
|
+
@connection.expects(:select_value).with(regexp_matches(/where id >= 1 order by id limit 1 offset 0/),EXPECTED_RETRY_FLAGS_CHUNKER).returns(nil)
|
108
|
+
@connection.expects(:update).with(regexp_matches(/between 1 and 1/),EXPECTED_RETRY_FLAGS_CHUNK_INSERT).returns(1)
|
108
109
|
|
109
110
|
@chunker.run
|
110
111
|
end
|
@@ -117,17 +118,17 @@ describe Lhm::Chunker do
|
|
117
118
|
2
|
118
119
|
end
|
119
120
|
|
120
|
-
@connection.expects(:select_value).with(regexp_matches(/where id >= 2 order by id limit 1 offset 1/),
|
121
|
-
@connection.expects(:select_value).with(regexp_matches(/where id >= 4 order by id limit 1 offset 1/),
|
122
|
-
@connection.expects(:select_value).with(regexp_matches(/where id >= 6 order by id limit 1 offset 1/),
|
123
|
-
@connection.expects(:select_value).with(regexp_matches(/where id >= 8 order by id limit 1 offset 1/),
|
124
|
-
@connection.expects(:select_value).with(regexp_matches(/where id >= 10 order by id limit 1 offset 1/),
|
121
|
+
@connection.expects(:select_value).with(regexp_matches(/where id >= 2 order by id limit 1 offset 1/),EXPECTED_RETRY_FLAGS_CHUNKER).returns(3)
|
122
|
+
@connection.expects(:select_value).with(regexp_matches(/where id >= 4 order by id limit 1 offset 1/),EXPECTED_RETRY_FLAGS_CHUNKER).returns(5)
|
123
|
+
@connection.expects(:select_value).with(regexp_matches(/where id >= 6 order by id limit 1 offset 1/),EXPECTED_RETRY_FLAGS_CHUNKER).returns(7)
|
124
|
+
@connection.expects(:select_value).with(regexp_matches(/where id >= 8 order by id limit 1 offset 1/),EXPECTED_RETRY_FLAGS_CHUNKER).returns(9)
|
125
|
+
@connection.expects(:select_value).with(regexp_matches(/where id >= 10 order by id limit 1 offset 1/),EXPECTED_RETRY_FLAGS_CHUNKER).returns(nil)
|
125
126
|
|
126
|
-
@connection.expects(:update).with(regexp_matches(/between 2 and 3/),
|
127
|
-
@connection.expects(:update).with(regexp_matches(/between 4 and 5/),
|
128
|
-
@connection.expects(:update).with(regexp_matches(/between 6 and 7/),
|
129
|
-
@connection.expects(:update).with(regexp_matches(/between 8 and 9/),
|
130
|
-
@connection.expects(:update).with(regexp_matches(/between 10 and 10/),
|
127
|
+
@connection.expects(:update).with(regexp_matches(/between 2 and 3/),EXPECTED_RETRY_FLAGS_CHUNK_INSERT).returns(2)
|
128
|
+
@connection.expects(:update).with(regexp_matches(/between 4 and 5/),EXPECTED_RETRY_FLAGS_CHUNK_INSERT).returns(2)
|
129
|
+
@connection.expects(:update).with(regexp_matches(/between 6 and 7/),EXPECTED_RETRY_FLAGS_CHUNK_INSERT).returns(2)
|
130
|
+
@connection.expects(:update).with(regexp_matches(/between 8 and 9/),EXPECTED_RETRY_FLAGS_CHUNK_INSERT).returns(2)
|
131
|
+
@connection.expects(:update).with(regexp_matches(/between 10 and 10/),EXPECTED_RETRY_FLAGS_CHUNK_INSERT).returns(1)
|
131
132
|
|
132
133
|
@chunker.run
|
133
134
|
end
|
@@ -141,9 +142,9 @@ describe Lhm::Chunker do
|
|
141
142
|
2
|
142
143
|
end
|
143
144
|
|
144
|
-
@connection.expects(:select_value).with(regexp_matches(/where id >= 1 order by id limit 1 offset 1/),
|
145
|
-
@connection.expects(:update).with(regexp_matches(/where \(foo.created_at > '2013-07-10' or foo.baz = 'quux'\) and `foo`/),
|
146
|
-
@connection.expects(:execute).with(regexp_matches(/show warnings/),
|
145
|
+
@connection.expects(:select_value).with(regexp_matches(/where id >= 1 order by id limit 1 offset 1/),EXPECTED_RETRY_FLAGS_CHUNKER).returns(2)
|
146
|
+
@connection.expects(:update).with(regexp_matches(/where \(foo.created_at > '2013-07-10' or foo.baz = 'quux'\) and `foo`/),EXPECTED_RETRY_FLAGS_CHUNK_INSERT).returns(1)
|
147
|
+
@connection.expects(:execute).with(regexp_matches(/show warnings/),EXPECTED_RETRY_FLAGS_CHUNKER).returns([])
|
147
148
|
|
148
149
|
def @migration.conditions
|
149
150
|
"where foo.created_at > '2013-07-10' or foo.baz = 'quux'"
|
@@ -161,9 +162,9 @@ describe Lhm::Chunker do
|
|
161
162
|
2
|
162
163
|
end
|
163
164
|
|
164
|
-
@connection.expects(:select_value).with(regexp_matches(/where id >= 1 order by id limit 1 offset 1/),
|
165
|
-
@connection.expects(:update).with(regexp_matches(/inner join bar on foo.id = bar.foo_id and/),
|
166
|
-
@connection.expects(:execute).with(regexp_matches(/show warnings/),
|
165
|
+
@connection.expects(:select_value).with(regexp_matches(/where id >= 1 order by id limit 1 offset 1/),EXPECTED_RETRY_FLAGS_CHUNKER).returns(2)
|
166
|
+
@connection.expects(:update).with(regexp_matches(/inner join bar on foo.id = bar.foo_id and/),EXPECTED_RETRY_FLAGS_CHUNK_INSERT).returns(1)
|
167
|
+
@connection.expects(:execute).with(regexp_matches(/show warnings/),EXPECTED_RETRY_FLAGS_CHUNKER).returns([])
|
167
168
|
|
168
169
|
def @migration.conditions
|
169
170
|
'inner join bar on foo.id = bar.foo_id'
|
@@ -15,9 +15,13 @@ describe Lhm::Connection do
|
|
15
15
|
ar_connection.stubs(:execute).raises(LOCK_WAIT).then.returns(true)
|
16
16
|
ar_connection.stubs(:active?).returns(true)
|
17
17
|
|
18
|
-
connection = Lhm::Connection.new(connection: ar_connection
|
18
|
+
connection = Lhm::Connection.new(connection: ar_connection, options: {
|
19
|
+
retriable: {
|
20
|
+
base_interval: 0
|
21
|
+
}
|
22
|
+
})
|
19
23
|
|
20
|
-
connection.execute("SHOW TABLES", should_retry: true
|
24
|
+
connection.execute("SHOW TABLES", should_retry: true)
|
21
25
|
|
22
26
|
log_messages = @logs.string.split("\n")
|
23
27
|
assert_equal(1, log_messages.length)
|
@@ -31,9 +35,14 @@ describe Lhm::Connection do
|
|
31
35
|
.then.returns(true)
|
32
36
|
ar_connection.stubs(:active?).returns(true)
|
33
37
|
|
34
|
-
connection = Lhm::Connection.new(connection: ar_connection
|
38
|
+
connection = Lhm::Connection.new(connection: ar_connection, options: {
|
39
|
+
retriable: {
|
40
|
+
base_interval: 0,
|
41
|
+
tries: 3
|
42
|
+
}
|
43
|
+
})
|
35
44
|
|
36
|
-
connection.execute("SHOW TABLES", should_retry: true
|
45
|
+
connection.execute("SHOW TABLES", should_retry: true)
|
37
46
|
|
38
47
|
log_messages = @logs.string.split("\n")
|
39
48
|
assert_equal(2, log_messages.length)
|
@@ -46,9 +55,14 @@ describe Lhm::Connection do
|
|
46
55
|
.then.returns(1)
|
47
56
|
ar_connection.stubs(:active?).returns(true)
|
48
57
|
|
49
|
-
connection = Lhm::Connection.new(connection: ar_connection
|
58
|
+
connection = Lhm::Connection.new(connection: ar_connection, options: {
|
59
|
+
retriable: {
|
60
|
+
base_interval: 0,
|
61
|
+
tries: 3
|
62
|
+
}
|
63
|
+
})
|
50
64
|
|
51
|
-
val = connection.update("SHOW TABLES", should_retry: true
|
65
|
+
val = connection.update("SHOW TABLES", should_retry: true)
|
52
66
|
|
53
67
|
log_messages = @logs.string.split("\n")
|
54
68
|
assert_equal val, 1
|
@@ -62,24 +76,35 @@ describe Lhm::Connection do
|
|
62
76
|
.then.returns("dummy")
|
63
77
|
ar_connection.stubs(:active?).returns(true)
|
64
78
|
|
65
|
-
connection = Lhm::Connection.new(connection: ar_connection
|
79
|
+
connection = Lhm::Connection.new(connection: ar_connection, options: {
|
80
|
+
retriable: {
|
81
|
+
base_interval: 0,
|
82
|
+
tries: 3
|
83
|
+
}
|
84
|
+
})
|
66
85
|
|
67
|
-
val = connection.select_value("SHOW TABLES", should_retry: true
|
86
|
+
val = connection.select_value("SHOW TABLES", should_retry: true)
|
68
87
|
|
69
88
|
log_messages = @logs.string.split("\n")
|
70
89
|
assert_equal val, "dummy"
|
71
90
|
assert_equal(2, log_messages.length)
|
72
91
|
end
|
73
92
|
|
74
|
-
it "Queries should be tagged with ProxySQL tag if
|
93
|
+
it "Queries should be tagged with ProxySQL tag if reconnect_with_consistent_host is enabled" do
|
75
94
|
ar_connection = mock()
|
76
95
|
ar_connection.expects(:public_send).with(:select_value, "SHOW TABLES #{Lhm::ProxySQLHelper::ANNOTATION}").returns("dummy")
|
77
96
|
ar_connection.stubs(:execute).times(4).returns([["dummy"]])
|
78
97
|
ar_connection.stubs(:active?).returns(true)
|
79
98
|
|
80
|
-
connection = Lhm::Connection.new(connection: ar_connection, options: {
|
99
|
+
connection = Lhm::Connection.new(connection: ar_connection, options: {
|
100
|
+
reconnect_with_consistent_host: true,
|
101
|
+
retriable: {
|
102
|
+
base_interval: 0,
|
103
|
+
tries: 3
|
104
|
+
}
|
105
|
+
})
|
81
106
|
|
82
|
-
val = connection.select_value("SHOW TABLES", should_retry: true
|
107
|
+
val = connection.select_value("SHOW TABLES", should_retry: true)
|
83
108
|
|
84
109
|
assert_equal val, "dummy"
|
85
110
|
end
|
data/spec/unit/entangler_spec.rb
CHANGED
@@ -69,9 +69,15 @@ describe Lhm::Entangler do
|
|
69
69
|
.raises(Mysql2::Error, 'Lock wait timeout exceeded; try restarting transaction')
|
70
70
|
ar_connection.stubs(:active?).returns(true)
|
71
71
|
|
72
|
-
connection = Lhm::Connection.new(connection: ar_connection, options: {
|
72
|
+
connection = Lhm::Connection.new(connection: ar_connection, options: {
|
73
|
+
reconnect_with_consistent_host: true,
|
74
|
+
retriable: {
|
75
|
+
base_interval: 0,
|
76
|
+
tries: tries
|
77
|
+
}
|
78
|
+
})
|
73
79
|
|
74
|
-
@entangler = Lhm::Entangler.new(@migration, connection
|
80
|
+
@entangler = Lhm::Entangler.new(@migration, connection)
|
75
81
|
|
76
82
|
assert_raises(Mysql2::Error) { @entangler.before }
|
77
83
|
end
|
@@ -83,9 +89,14 @@ describe Lhm::Entangler do
|
|
83
89
|
.then
|
84
90
|
.raises(Mysql2::Error, 'The MySQL server is running with the --read-only option so it cannot execute this statement.')
|
85
91
|
ar_connection.stubs(:active?).returns(true)
|
86
|
-
connection = Lhm::Connection.new(connection: ar_connection, options: {
|
87
|
-
|
88
|
-
|
92
|
+
connection = Lhm::Connection.new(connection: ar_connection, options: {
|
93
|
+
reconnect_with_consistent_host: true,
|
94
|
+
retriable: {
|
95
|
+
base_interval: 0
|
96
|
+
},
|
97
|
+
})
|
98
|
+
|
99
|
+
@entangler = Lhm::Entangler.new(@migration, connection)
|
89
100
|
assert_raises(Mysql2::Error) { @entangler.before }
|
90
101
|
end
|
91
102
|
|
@@ -99,9 +110,14 @@ describe Lhm::Entangler do
|
|
99
110
|
.returns([["dummy"]])
|
100
111
|
ar_connection.stubs(:active?).returns(true)
|
101
112
|
|
102
|
-
connection = Lhm::Connection.new(connection: ar_connection, options: {
|
113
|
+
connection = Lhm::Connection.new(connection: ar_connection, options: {
|
114
|
+
reconnect_with_consistent_host: true,
|
115
|
+
retriable: {
|
116
|
+
base_interval: 0
|
117
|
+
},
|
118
|
+
})
|
103
119
|
|
104
|
-
@entangler = Lhm::Entangler.new(@migration, connection
|
120
|
+
@entangler = Lhm::Entangler.new(@migration, connection)
|
105
121
|
|
106
122
|
assert @entangler.before
|
107
123
|
end
|
@@ -126,9 +142,15 @@ describe Lhm::Entangler do
|
|
126
142
|
.raises(Mysql2::Error, 'Lock wait timeout exceeded; try restarting transaction') # final error
|
127
143
|
ar_connection.stubs(:active?).returns(true)
|
128
144
|
|
129
|
-
connection = Lhm::Connection.new(connection: ar_connection, options: {
|
145
|
+
connection = Lhm::Connection.new(connection: ar_connection, options: {
|
146
|
+
reconnect_with_consistent_host: true,
|
147
|
+
retriable: {
|
148
|
+
tries: 5,
|
149
|
+
base_interval: 0
|
150
|
+
},
|
151
|
+
})
|
130
152
|
|
131
|
-
@entangler = Lhm::Entangler.new(@migration, connection
|
153
|
+
@entangler = Lhm::Entangler.new(@migration, connection)
|
132
154
|
|
133
155
|
assert_raises(Mysql2::Error) { @entangler.before }
|
134
156
|
end
|
@@ -142,7 +142,7 @@ describe Lhm::Throttler::Slave do
|
|
142
142
|
Lhm::Throttler::Slave.any_instance.stubs(:config).returns([])
|
143
143
|
|
144
144
|
slave = Lhm::Throttler::Slave.new('slave', @dummy_mysql_client_config)
|
145
|
-
|
145
|
+
Logger.any_instance.expects(:info).with("Unable to connect and/or query slave: Can't connect to MySQL server")
|
146
146
|
assert_equal(0, slave.lag)
|
147
147
|
end
|
148
148
|
end
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: lhm-shopify
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 3.5.
|
4
|
+
version: 3.5.4
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- SoundCloud
|
@@ -12,7 +12,7 @@ authors:
|
|
12
12
|
autorequire:
|
13
13
|
bindir: bin
|
14
14
|
cert_chain: []
|
15
|
-
date: 2021-12-
|
15
|
+
date: 2021-12-14 00:00:00.000000000 Z
|
16
16
|
dependencies:
|
17
17
|
- !ruby/object:Gem::Dependency
|
18
18
|
name: retriable
|