lhm-shopify 3.5.3 → 3.5.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +8 -0
- data/Gemfile.lock +1 -1
- data/README.md +5 -3
- data/Rakefile +6 -5
- data/dev.yml +2 -2
- data/gemfiles/activerecord_5.2.gemfile.lock +1 -1
- data/gemfiles/activerecord_6.0.gemfile.lock +1 -1
- data/gemfiles/activerecord_6.1.gemfile.lock +1 -1
- data/gemfiles/activerecord_7.0.0.alpha2.gemfile.lock +1 -1
- data/lib/lhm/atomic_switcher.rb +4 -3
- data/lib/lhm/chunk_insert.rb +4 -1
- data/lib/lhm/chunker.rb +6 -6
- data/lib/lhm/cleanup/current.rb +4 -1
- data/lib/lhm/connection.rb +24 -25
- data/lib/lhm/entangler.rb +5 -4
- data/lib/lhm/invoker.rb +5 -3
- data/lib/lhm/locked_switcher.rb +2 -0
- data/lib/lhm/sql_retry.rb +19 -13
- data/lib/lhm/version.rb +1 -1
- data/lib/lhm.rb +28 -23
- data/spec/integration/atomic_switcher_spec.rb +28 -17
- data/spec/integration/chunker_spec.rb +7 -5
- data/spec/integration/integration_helper.rb +4 -6
- data/spec/integration/lhm_spec.rb +3 -3
- data/spec/unit/chunker_spec.rb +44 -43
- data/spec/unit/connection_spec.rb +36 -11
- data/spec/unit/entangler_spec.rb +31 -9
- data/spec/unit/throttler/slave_lag_spec.rb +1 -1
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 4f676427b10052113b08c4dda8f648bc11234024deab2f1e96de332e0906a110
|
4
|
+
data.tar.gz: dbd91d1b72d713e9d6e18edb46900971ee89c20563d5e93db9eec746887a12d7
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: be5dc9f5d6d2a3ae62b476fa19455bd0f864b989d6ffbff35a93a0afea4e8d9be4492a4b3bfc2bec1029193114187eaf4eda5bb6dfde9d0be34884bde317c1c1
|
7
|
+
data.tar.gz: aa4e866f74da3255586c1597ae63a3792606bd092e46c3a90335d22b9f6c9b243bc133d99e50233c884b03064dfaeee75983597f84733484e05ec456833f3395
|
data/CHANGELOG.md
CHANGED
@@ -1,3 +1,11 @@
|
|
1
|
+
# 3.5.4 (Dec, 2021)
|
2
|
+
* Refactored the way options are handled internally. Code is now much clearer to understand
|
3
|
+
* Removed optional connection_options from `Lhm.setup` and `Lhm.connection`
|
4
|
+
* Option `reconnect_with_consistent_host` will now be provided with `options` for `Lhm.change_table`
|
5
|
+
|
6
|
+
# 3.5.3 (Dec, 2021)
|
7
|
+
* Adds ProxySQL comments at the end of query to accommodate for internal tool's requirements
|
8
|
+
|
1
9
|
# 3.5.2 (Dec, 2021)
|
2
10
|
* Fixed error on undefined connection, when calling `Lhm.connection` without calling `Lhm.setup` first
|
3
11
|
* Changed `Lhm.connection.connection` to `lhm.connection.ar_connection` for increased clarity and readability
|
data/Gemfile.lock
CHANGED
data/README.md
CHANGED
@@ -113,7 +113,7 @@ tables must be cleaned up.
|
|
113
113
|
LHM can recover from connection loss. However, when used in conjunction with ProxySQL, there are multiple ways that
|
114
114
|
connection loss could induce data loss (if triggered by a failover). Therefore it will perform additional checks to
|
115
115
|
ensure that the MySQL host stays consistent across the schema migrations if the feature is enabled.
|
116
|
-
This is done by tagging every query with `/*maintenance:lhm*/`, which will be recognized by ProxySQL.
|
116
|
+
This is done by tagging every query with `/*maintenance:lhm*/`, which will be recognized by ProxySQL.
|
117
117
|
However, to get this feature working, a new ProxySQL query rule must be added.
|
118
118
|
```cnf
|
119
119
|
{
|
@@ -145,9 +145,11 @@ forwarded to the right target.
|
|
145
145
|
```
|
146
146
|
|
147
147
|
Once these changes are added to the ProxySQL configuration (either through `.cnf` or dynamically through the admin interface),
|
148
|
-
the feature can be enabled. This is done by adding this flag when
|
148
|
+
the feature can be enabled. This is done by adding this flag when providing options to the migration:
|
149
149
|
```ruby
|
150
|
-
Lhm.
|
150
|
+
Lhm.change_table(..., options: {reconnect_with_consistent_host: true}) do |t|
|
151
|
+
...
|
152
|
+
end
|
151
153
|
```
|
152
154
|
**Note**: This feature is disabled by default
|
153
155
|
|
data/Rakefile
CHANGED
@@ -16,15 +16,16 @@ Rake::TestTask.new('integration') do |t|
|
|
16
16
|
t.libs << 'spec'
|
17
17
|
t.test_files = FileList['spec/integration/**/*_spec.rb']
|
18
18
|
t.verbose = true
|
19
|
-
|
19
|
+
end
|
20
20
|
|
21
21
|
Rake::TestTask.new('dev') do |t|
|
22
22
|
t.libs << 'lib'
|
23
23
|
t.libs << 'spec'
|
24
|
-
|
25
|
-
|
26
|
-
|
27
|
-
|
24
|
+
|
25
|
+
files = FileList.new('spec/test_helper.rb')
|
26
|
+
files.add(ENV["SINGLE_TEST"]) if ENV["SINGLE_TEST"]
|
27
|
+
t.test_files = files
|
28
|
+
|
28
29
|
t.verbose = true
|
29
30
|
end
|
30
31
|
|
data/dev.yml
CHANGED
@@ -5,7 +5,7 @@ up:
|
|
5
5
|
or: [mysql@5.7]
|
6
6
|
conflicts: [shopify/shopify/mysql-client, mysql-connector-c, mysql, mysql-client]
|
7
7
|
- wget
|
8
|
-
- ruby: 2.7.
|
8
|
+
- ruby: 2.7.5
|
9
9
|
- bundler
|
10
10
|
- custom:
|
11
11
|
name: Get Appraisal gems
|
@@ -31,7 +31,7 @@ commands:
|
|
31
31
|
if [[ $# -eq 0 ]]; then
|
32
32
|
bundle exec rake unit && bundle exec rake integration
|
33
33
|
else
|
34
|
-
bundle exec rake dev
|
34
|
+
SINGLE_TEST="$@" bundle exec rake dev
|
35
35
|
fi
|
36
36
|
appraisals: bundle exec appraisal rake specs
|
37
37
|
cov: rm -rf coverage; COV=1 bundle exec rake unit && bundle exec rake integration; open coverage/index.html
|
data/lib/lhm/atomic_switcher.rb
CHANGED
@@ -16,12 +16,13 @@ module Lhm
|
|
16
16
|
|
17
17
|
attr_reader :connection
|
18
18
|
|
19
|
-
|
19
|
+
LOG_PREFIX = "AtomicSwitcher"
|
20
|
+
|
21
|
+
def initialize(migration, connection = nil)
|
20
22
|
@migration = migration
|
21
23
|
@connection = connection
|
22
24
|
@origin = migration.origin
|
23
25
|
@destination = migration.destination
|
24
|
-
@retry_options = options[:retriable] || {}
|
25
26
|
end
|
26
27
|
|
27
28
|
def atomic_switch
|
@@ -39,7 +40,7 @@ module Lhm
|
|
39
40
|
private
|
40
41
|
|
41
42
|
def execute
|
42
|
-
@connection.execute(atomic_switch, should_retry: true,
|
43
|
+
@connection.execute(atomic_switch, should_retry: true, log_prefix: LOG_PREFIX)
|
43
44
|
end
|
44
45
|
end
|
45
46
|
end
|
data/lib/lhm/chunk_insert.rb
CHANGED
@@ -3,6 +3,9 @@ require 'lhm/proxysql_helper'
|
|
3
3
|
|
4
4
|
module Lhm
|
5
5
|
class ChunkInsert
|
6
|
+
|
7
|
+
LOG_PREFIX = "ChunkInsert"
|
8
|
+
|
6
9
|
def initialize(migration, connection, lowest, highest, retry_options = {})
|
7
10
|
@migration = migration
|
8
11
|
@connection = connection
|
@@ -12,7 +15,7 @@ module Lhm
|
|
12
15
|
end
|
13
16
|
|
14
17
|
def insert_and_return_count_of_rows_created
|
15
|
-
@connection.update(sql, should_retry: true,
|
18
|
+
@connection.update(sql, should_retry: true, log_prefix: LOG_PREFIX)
|
16
19
|
end
|
17
20
|
|
18
21
|
def sql
|
data/lib/lhm/chunker.rb
CHANGED
@@ -13,6 +13,8 @@ module Lhm
|
|
13
13
|
|
14
14
|
attr_reader :connection
|
15
15
|
|
16
|
+
LOG_PREFIX = "Chunker"
|
17
|
+
|
16
18
|
# Copy from origin to destination in chunks of size `stride`.
|
17
19
|
# Use the `throttler` class to sleep between each stride.
|
18
20
|
def initialize(migration, connection = nil, options = {})
|
@@ -31,9 +33,7 @@ module Lhm
|
|
31
33
|
@retry_options = options[:retriable] || {}
|
32
34
|
@retry_helper = SqlRetry.new(
|
33
35
|
@connection,
|
34
|
-
retry_options:
|
35
|
-
log_prefix: "Chunker"
|
36
|
-
}.merge!(@retry_options)
|
36
|
+
retry_options: @retry_options
|
37
37
|
)
|
38
38
|
end
|
39
39
|
|
@@ -79,7 +79,7 @@ module Lhm
|
|
79
79
|
private
|
80
80
|
|
81
81
|
def raise_on_non_pk_duplicate_warning
|
82
|
-
@connection.execute("show warnings", should_retry: true,
|
82
|
+
@connection.execute("show warnings", should_retry: true, log_prefix: LOG_PREFIX).each do |level, code, message|
|
83
83
|
unless message.match?(/Duplicate entry .+ for key 'PRIMARY'/)
|
84
84
|
m = "Unexpected warning found for inserted row: #{message}"
|
85
85
|
Lhm.logger.warn(m)
|
@@ -94,14 +94,14 @@ module Lhm
|
|
94
94
|
|
95
95
|
def verify_can_run
|
96
96
|
return unless @verifier
|
97
|
-
@retry_helper.with_retries(
|
97
|
+
@retry_helper.with_retries(log_prefix: LOG_PREFIX) do |retriable_connection|
|
98
98
|
raise "Verification failed, aborting early" if !@verifier.call(retriable_connection)
|
99
99
|
end
|
100
100
|
end
|
101
101
|
|
102
102
|
def upper_id(next_id, stride)
|
103
103
|
sql = "select id from `#{ @migration.origin_name }` where id >= #{ next_id } order by id limit 1 offset #{ stride - 1}"
|
104
|
-
top = @connection.select_value(sql, should_retry: true,
|
104
|
+
top = @connection.select_value(sql, should_retry: true, log_prefix: LOG_PREFIX)
|
105
105
|
|
106
106
|
[top ? top.to_i : @limit, @limit].min
|
107
107
|
end
|
data/lib/lhm/cleanup/current.rb
CHANGED
@@ -4,6 +4,9 @@ require 'lhm/sql_retry'
|
|
4
4
|
module Lhm
|
5
5
|
module Cleanup
|
6
6
|
class Current
|
7
|
+
|
8
|
+
LOG_PREFIX = "Current"
|
9
|
+
|
7
10
|
def initialize(run, origin_table_name, connection, options={})
|
8
11
|
@run = run
|
9
12
|
@table_name = TableName.new(origin_table_name)
|
@@ -54,7 +57,7 @@ module Lhm
|
|
54
57
|
|
55
58
|
def execute_ddls
|
56
59
|
ddls.each do |ddl|
|
57
|
-
@connection.execute(ddl, should_retry: true,
|
60
|
+
@connection.execute(ddl, should_retry: true, log_prefix: LOG_PREFIX)
|
58
61
|
end
|
59
62
|
Lhm.logger.info("Dropped triggers on #{@lhm_triggers_for_origin.join(', ')}")
|
60
63
|
Lhm.logger.info("Dropped tables #{@lhm_triggers_for_origin.join(', ')}")
|
data/lib/lhm/connection.rb
CHANGED
@@ -1,13 +1,20 @@
|
|
1
1
|
require 'delegate'
|
2
|
+
require 'forwardable'
|
2
3
|
require 'lhm/sql_retry'
|
3
4
|
|
4
5
|
module Lhm
|
5
6
|
# Lhm::Connection inherits from SingleDelegator. It will forward any unknown method calls to the ActiveRecord
|
6
7
|
# connection.
|
7
8
|
class Connection < SimpleDelegator
|
9
|
+
extend Forwardable
|
8
10
|
|
9
|
-
|
10
|
-
|
11
|
+
# Will delegate the following function to @sql_retry object, while leaving them accessible from the Lhm::Connection
|
12
|
+
# object
|
13
|
+
def_delegators :@sql_retry, :reconnect_with_consistent_host, :reconnect_with_consistent_host=, :retry_config=
|
14
|
+
|
15
|
+
alias ar_connection __getobj__
|
16
|
+
|
17
|
+
def initialize(connection:, options: {})
|
11
18
|
@sql_retry = Lhm::SqlRetry.new(
|
12
19
|
connection,
|
13
20
|
retry_options: options[:retriable] || {},
|
@@ -18,11 +25,6 @@ module Lhm
|
|
18
25
|
super(connection)
|
19
26
|
end
|
20
27
|
|
21
|
-
def ar_connection
|
22
|
-
# Get object from the simple delegator
|
23
|
-
__getobj__
|
24
|
-
end
|
25
|
-
|
26
28
|
def ar_connection=(connection)
|
27
29
|
raise Lhm::Error.new("Lhm::Connection requires an active record connection to operate") if connection.nil?
|
28
30
|
|
@@ -31,46 +33,43 @@ module Lhm
|
|
31
33
|
__setobj__(connection)
|
32
34
|
end
|
33
35
|
|
34
|
-
|
35
|
-
|
36
|
-
|
37
|
-
end
|
38
|
-
|
39
|
-
def execute(query, should_retry: false, retry_options: {})
|
36
|
+
# ActiveRecord::Base overridden methods to incorporate custom retry logic
|
37
|
+
# All other methods will be delegated
|
38
|
+
def execute(query, should_retry: false, log_prefix: nil)
|
40
39
|
if should_retry
|
41
|
-
exec_with_retries(:execute, query,
|
40
|
+
exec_with_retries(:execute, query, log_prefix)
|
42
41
|
else
|
43
42
|
exec(:execute, query)
|
44
43
|
end
|
45
44
|
end
|
46
45
|
|
47
|
-
def update(query, should_retry: false,
|
46
|
+
def update(query, should_retry: false, log_prefix: nil)
|
48
47
|
if should_retry
|
49
|
-
exec_with_retries(:update, query,
|
48
|
+
exec_with_retries(:update, query, log_prefix)
|
50
49
|
else
|
51
50
|
exec(:update, query)
|
52
51
|
end
|
53
52
|
end
|
54
53
|
|
55
|
-
def select_value(query, should_retry: false,
|
54
|
+
def select_value(query, should_retry: false, log_prefix: nil)
|
56
55
|
if should_retry
|
57
|
-
exec_with_retries(:select_value, query,
|
56
|
+
exec_with_retries(:select_value, query, log_prefix)
|
58
57
|
else
|
59
58
|
exec(:select_value, query)
|
60
59
|
end
|
61
60
|
end
|
62
61
|
|
63
|
-
def select_values(query, should_retry: false,
|
62
|
+
def select_values(query, should_retry: false, log_prefix: nil)
|
64
63
|
if should_retry
|
65
|
-
exec_with_retries(:select_values, query,
|
64
|
+
exec_with_retries(:select_values, query, log_prefix)
|
66
65
|
else
|
67
66
|
exec(:select_values, query)
|
68
67
|
end
|
69
68
|
end
|
70
69
|
|
71
|
-
def select_one(query, should_retry: false,
|
70
|
+
def select_one(query, should_retry: false, log_prefix: nil)
|
72
71
|
if should_retry
|
73
|
-
exec_with_retries(:select_one, query,
|
72
|
+
exec_with_retries(:select_one, query, log_prefix)
|
74
73
|
else
|
75
74
|
exec(:select_one, query)
|
76
75
|
end
|
@@ -82,9 +81,9 @@ module Lhm
|
|
82
81
|
ar_connection.public_send(method, Lhm::ProxySQLHelper.tagged(sql))
|
83
82
|
end
|
84
83
|
|
85
|
-
def exec_with_retries(method, sql,
|
86
|
-
|
87
|
-
@sql_retry.with_retries(
|
84
|
+
def exec_with_retries(method, sql, log_prefix=nil)
|
85
|
+
effective_log_prefix = log_prefix || file
|
86
|
+
@sql_retry.with_retries(log_prefix: effective_log_prefix) do |conn|
|
88
87
|
conn.public_send(method, Lhm::ProxySQLHelper.tagged(sql))
|
89
88
|
end
|
90
89
|
end
|
data/lib/lhm/entangler.rb
CHANGED
@@ -13,14 +13,15 @@ module Lhm
|
|
13
13
|
|
14
14
|
attr_reader :connection
|
15
15
|
|
16
|
+
LOG_PREFIX = "Entangler"
|
17
|
+
|
16
18
|
# Creates entanglement between two tables. All creates, updates and deletes
|
17
19
|
# to origin will be repeated on the destination table.
|
18
|
-
def initialize(migration, connection = nil
|
20
|
+
def initialize(migration, connection = nil)
|
19
21
|
@intersection = migration.intersection
|
20
22
|
@origin = migration.origin
|
21
23
|
@destination = migration.destination
|
22
24
|
@connection = connection
|
23
|
-
@retry_options = options[:retriable] || {}
|
24
25
|
end
|
25
26
|
|
26
27
|
def entangle
|
@@ -86,14 +87,14 @@ module Lhm
|
|
86
87
|
|
87
88
|
def before
|
88
89
|
entangle.each do |stmt|
|
89
|
-
@connection.execute(stmt, should_retry: true,
|
90
|
+
@connection.execute(stmt, should_retry: true, log_prefix: LOG_PREFIX)
|
90
91
|
end
|
91
92
|
Lhm.logger.info("Created triggers on #{@origin.name}")
|
92
93
|
end
|
93
94
|
|
94
95
|
def after
|
95
96
|
untangle.each do |stmt|
|
96
|
-
@connection.execute(stmt, should_retry: true,
|
97
|
+
@connection.execute(stmt, should_retry: true, log_prefix: LOG_PREFIX)
|
97
98
|
end
|
98
99
|
Lhm.logger.info("Dropped triggers on #{@origin.name}")
|
99
100
|
end
|
data/lib/lhm/invoker.rb
CHANGED
@@ -16,8 +16,8 @@ module Lhm
|
|
16
16
|
class Invoker
|
17
17
|
include SqlHelper
|
18
18
|
LOCK_WAIT_TIMEOUT_DELTA = 10
|
19
|
-
INNODB_LOCK_WAIT_TIMEOUT_MAX=1073741824.freeze # https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_lock_wait_timeout
|
20
|
-
LOCK_WAIT_TIMEOUT_MAX=31536000.freeze # https://dev.mysql.com/doc/refman/5.7/en/server-system-variables.html
|
19
|
+
INNODB_LOCK_WAIT_TIMEOUT_MAX = 1073741824.freeze # https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_lock_wait_timeout
|
20
|
+
LOCK_WAIT_TIMEOUT_MAX = 31536000.freeze # https://dev.mysql.com/doc/refman/5.7/en/server-system-variables.html
|
21
21
|
|
22
22
|
attr_reader :migrator, :connection
|
23
23
|
|
@@ -49,7 +49,7 @@ module Lhm
|
|
49
49
|
normalize_options(options)
|
50
50
|
set_session_lock_wait_timeouts
|
51
51
|
migration = @migrator.run
|
52
|
-
entangler = Entangler.new(migration, @connection
|
52
|
+
entangler = Entangler.new(migration, @connection)
|
53
53
|
|
54
54
|
entangler.run do
|
55
55
|
options[:verifier] ||= Proc.new { |conn| triggers_still_exist?(conn, entangler) }
|
@@ -90,6 +90,8 @@ module Lhm
|
|
90
90
|
options[:throttler] = Lhm.throttler
|
91
91
|
end
|
92
92
|
|
93
|
+
Lhm.connection.retry_config = options[:retriable] || {}
|
94
|
+
|
93
95
|
rescue => e
|
94
96
|
Lhm.logger.error "LHM run failed with exception=#{e.class} message=#{e.message}"
|
95
97
|
raise
|
data/lib/lhm/locked_switcher.rb
CHANGED
data/lib/lhm/sql_retry.rb
CHANGED
@@ -30,21 +30,15 @@ module Lhm
|
|
30
30
|
|
31
31
|
def initialize(connection, retry_options: {}, reconnect_with_consistent_host: false)
|
32
32
|
@connection = connection
|
33
|
-
|
34
|
-
|
35
|
-
if (@reconnect_with_consistent_host = reconnect_with_consistent_host)
|
36
|
-
@initial_hostname = hostname
|
37
|
-
@initial_server_id = server_id
|
38
|
-
end
|
33
|
+
self.retry_config = retry_options
|
34
|
+
self.reconnect_with_consistent_host = reconnect_with_consistent_host
|
39
35
|
end
|
40
36
|
|
41
37
|
# Complete explanation of algorithm: https://github.com/Shopify/lhm/pull/112
|
42
|
-
def with_retries(
|
43
|
-
@log_prefix =
|
38
|
+
def with_retries(log_prefix: nil)
|
39
|
+
@log_prefix = log_prefix || "" # No prefix. Just logs
|
44
40
|
|
45
|
-
|
46
|
-
|
47
|
-
Retriable.retriable(retry_config) do
|
41
|
+
Retriable.retriable(@retry_config) do
|
48
42
|
# Using begin -> rescue -> end for Ruby 2.4 compatibility
|
49
43
|
begin
|
50
44
|
if @reconnect_with_consistent_host
|
@@ -62,8 +56,20 @@ module Lhm
|
|
62
56
|
end
|
63
57
|
end
|
64
58
|
|
65
|
-
|
66
|
-
|
59
|
+
# Both attributes will have defined setters
|
60
|
+
attr_reader :retry_config, :reconnect_with_consistent_host
|
61
|
+
attr_accessor :connection
|
62
|
+
|
63
|
+
def retry_config=(retry_options)
|
64
|
+
@retry_config = default_retry_config.dup.merge!(retry_options)
|
65
|
+
end
|
66
|
+
|
67
|
+
def reconnect_with_consistent_host=(reconnect)
|
68
|
+
if (@reconnect_with_consistent_host = reconnect)
|
69
|
+
@initial_hostname = hostname
|
70
|
+
@initial_server_id = server_id
|
71
|
+
end
|
72
|
+
end
|
67
73
|
|
68
74
|
private
|
69
75
|
|
data/lib/lhm/version.rb
CHANGED
data/lib/lhm.rb
CHANGED
@@ -46,15 +46,19 @@ module Lhm
|
|
46
46
|
# Use atomic switch to rename tables (defaults to: true)
|
47
47
|
# If using a version of mysql affected by atomic switch bug, LHM forces user
|
48
48
|
# to set this option (see SqlHelper#supports_atomic_switch?)
|
49
|
+
# @option options [Boolean] :reconnect_with_consistent_host
|
50
|
+
# Active / Deactivate ProxySQL-aware reconnection procedure (default to: false)
|
49
51
|
# @yield [Migrator] Yielded Migrator object records the changes
|
50
52
|
# @return [Boolean] Returns true if the migration finishes
|
51
53
|
# @raise [Error] Raises Lhm::Error in case of a error and aborts the migration
|
52
54
|
def change_table(table_name, options = {}, &block)
|
53
|
-
|
54
|
-
|
55
|
-
|
56
|
-
|
57
|
-
|
55
|
+
with_flags(options) do
|
56
|
+
origin = Table.parse(table_name, connection)
|
57
|
+
invoker = Invoker.new(origin, connection)
|
58
|
+
block.call(invoker.migrator)
|
59
|
+
invoker.run(options)
|
60
|
+
true
|
61
|
+
end
|
58
62
|
end
|
59
63
|
|
60
64
|
# Cleanup tables and triggers
|
@@ -86,27 +90,16 @@ module Lhm
|
|
86
90
|
# Setups DB connection
|
87
91
|
#
|
88
92
|
# @param [ActiveRecord::Base] connection ActiveRecord Connection
|
89
|
-
|
90
|
-
|
91
|
-
# Active / Deactivate ProxySQL-aware reconnection procedure (default to: false)
|
92
|
-
def setup(connection, connection_options = {})
|
93
|
-
@@connection = Connection.new(connection: connection, options: connection_options)
|
93
|
+
def setup(connection)
|
94
|
+
@@connection = Connection.new(connection: connection)
|
94
95
|
end
|
95
96
|
|
96
|
-
#
|
97
|
-
|
98
|
-
# @param [Hash] connection_options Optional options (defaults to: empty hash)
|
99
|
-
# @option connection_options [Boolean] :reconnect_with_consistent_host
|
100
|
-
# Active / Deactivate ProxySQL-aware reconnection procedure (default to: false)
|
101
|
-
def connection(connection_options = nil)
|
97
|
+
# Returns DB connection (or initializes it if not created yet)
|
98
|
+
def connection
|
102
99
|
@@connection ||= begin
|
103
|
-
|
104
|
-
|
105
|
-
|
106
|
-
|
107
|
-
@@connection.process_connection_options(connection_options) unless connection_options.nil?
|
108
|
-
|
109
|
-
@@connection
|
100
|
+
raise 'Please call Lhm.setup' unless defined?(ActiveRecord)
|
101
|
+
@@connection = Connection.new(connection: ActiveRecord::Base.connection)
|
102
|
+
end
|
110
103
|
end
|
111
104
|
|
112
105
|
def self.logger=(new_logger)
|
@@ -148,4 +141,16 @@ module Lhm
|
|
148
141
|
false
|
149
142
|
end
|
150
143
|
end
|
144
|
+
|
145
|
+
def with_flags(options)
|
146
|
+
old_flags = {
|
147
|
+
reconnect_with_consistent_host: Lhm.connection.reconnect_with_consistent_host,
|
148
|
+
}
|
149
|
+
|
150
|
+
Lhm.connection.reconnect_with_consistent_host = options[:reconnect_with_consistent_host] || false
|
151
|
+
|
152
|
+
yield
|
153
|
+
ensure
|
154
|
+
Lhm.connection.reconnect_with_consistent_host = old_flags[:reconnect_with_consistent_host]
|
155
|
+
end
|
151
156
|
end
|
@@ -39,10 +39,15 @@ describe Lhm::AtomicSwitcher do
|
|
39
39
|
.then
|
40
40
|
.returns([["dummy"]]) # Matches initial host -> triggers retry
|
41
41
|
|
42
|
-
connection = Lhm::Connection.new(connection: ar_connection, options: {
|
42
|
+
connection = Lhm::Connection.new(connection: ar_connection, options: {
|
43
|
+
reconnect_with_consistent_host: true,
|
44
|
+
retriable: {
|
45
|
+
tries: 3,
|
46
|
+
base_interval: 0
|
47
|
+
}
|
48
|
+
})
|
43
49
|
|
44
|
-
|
45
|
-
switcher = Lhm::AtomicSwitcher.new(@migration, connection, retriable: { tries: 3, base_interval: 0 })
|
50
|
+
switcher = Lhm::AtomicSwitcher.new(@migration, connection)
|
46
51
|
|
47
52
|
assert switcher.run
|
48
53
|
|
@@ -58,20 +63,26 @@ describe Lhm::AtomicSwitcher do
|
|
58
63
|
ar_connection.stubs(:data_source_exists?).returns(true)
|
59
64
|
ar_connection.stubs(:active?).returns(true)
|
60
65
|
ar_connection.stubs(:execute).returns([["dummy"]], [["dummy"]], [["dummy"]])
|
61
|
-
|
62
|
-
|
63
|
-
|
64
|
-
|
65
|
-
|
66
|
-
|
67
|
-
|
68
|
-
|
69
|
-
|
70
|
-
|
71
|
-
|
72
|
-
connection = Lhm::Connection.new(connection: ar_connection, options: {
|
73
|
-
|
74
|
-
|
66
|
+
.then
|
67
|
+
.raises(ActiveRecord::StatementInvalid, 'Lock wait timeout exceeded; try restarting transaction.')
|
68
|
+
.then
|
69
|
+
.returns([["dummy"]]) # triggers retry 1
|
70
|
+
.then
|
71
|
+
.raises(ActiveRecord::StatementInvalid, 'Lock wait timeout exceeded; try restarting transaction.')
|
72
|
+
.then
|
73
|
+
.returns([["dummy"]]) # triggers retry 2
|
74
|
+
.then
|
75
|
+
.raises(ActiveRecord::StatementInvalid, 'Lock wait timeout exceeded; try restarting transaction.') # triggers retry 2
|
76
|
+
|
77
|
+
connection = Lhm::Connection.new(connection: ar_connection, options: {
|
78
|
+
reconnect_with_consistent_host: true,
|
79
|
+
retriable: {
|
80
|
+
tries: 2,
|
81
|
+
base_interval: 0
|
82
|
+
}
|
83
|
+
})
|
84
|
+
|
85
|
+
switcher = Lhm::AtomicSwitcher.new(@migration, connection)
|
75
86
|
|
76
87
|
assert_raises(ActiveRecord::StatementInvalid) { switcher.run }
|
77
88
|
end
|
@@ -173,6 +173,9 @@ describe Lhm::Chunker do
|
|
173
173
|
printer.expect(:notify, :return_value, [Integer, Integer])
|
174
174
|
printer.expect(:end, :return_value, [])
|
175
175
|
|
176
|
+
Lhm::Throttler::Slave.any_instance.stubs(:slave_hosts).returns(['127.0.0.1'])
|
177
|
+
Lhm::Throttler::SlaveLag.any_instance.stubs(:master_slave_hosts).returns(['127.0.0.1'])
|
178
|
+
|
176
179
|
Lhm::Chunker.new(
|
177
180
|
@migration, connection, { throttler: Lhm::Throttler::SlaveLag.new(stride: 100), printer: printer }
|
178
181
|
).run
|
@@ -215,17 +218,16 @@ describe Lhm::Chunker do
|
|
215
218
|
printer.expects(:verify)
|
216
219
|
printer.expects(:end)
|
217
220
|
|
218
|
-
|
221
|
+
Lhm::Throttler::Slave.any_instance.stubs(:slave_hosts).returns(['127.0.0.1'])
|
222
|
+
Lhm::Throttler::SlaveLag.any_instance.stubs(:master_slave_hosts).returns(['127.0.0.1'])
|
219
223
|
|
220
|
-
|
221
|
-
['127.0.0.1']
|
222
|
-
end
|
224
|
+
throttler = Lhm::Throttler::SlaveLag.new(stride: 10, allowed_lag: 0)
|
223
225
|
|
224
226
|
if master_slave_mode?
|
225
227
|
def throttler.slave_connection(slave)
|
226
228
|
config = ActiveRecord::Base.connection_pool.db_config.configuration_hash.dup
|
227
229
|
config[:host] = slave
|
228
|
-
config[:port] =
|
230
|
+
config[:port] = 33007
|
229
231
|
ActiveRecord::Base.send('mysql2_connection', config)
|
230
232
|
end
|
231
233
|
end
|
@@ -62,18 +62,16 @@ module IntegrationHelper
|
|
62
62
|
)
|
63
63
|
end
|
64
64
|
|
65
|
-
def connect_master_with_toxiproxy!
|
65
|
+
def connect_master_with_toxiproxy!
|
66
66
|
connect!(
|
67
67
|
'127.0.0.1',
|
68
68
|
$db_config['master_toxic']['port'],
|
69
69
|
$db_config['master_toxic']['user'],
|
70
|
-
$db_config['master_toxic']['password']
|
71
|
-
with_retry)
|
70
|
+
$db_config['master_toxic']['password'])
|
72
71
|
end
|
73
72
|
|
74
|
-
def connect!(hostname, port, user, password
|
75
|
-
|
76
|
-
Lhm.setup(adapter,{reconnect_with_consistent_host: with_retry})
|
73
|
+
def connect!(hostname, port, user, password)
|
74
|
+
Lhm.setup(ar_conn(hostname, port, user, password))
|
77
75
|
unless defined?(@@cleaned_up)
|
78
76
|
Lhm.cleanup(true)
|
79
77
|
@@cleaned_up = true
|
@@ -592,7 +592,7 @@ describe Lhm do
|
|
592
592
|
end
|
593
593
|
|
594
594
|
it " should not try to reconnect if reconnect_with_consistent_host is not provided" do
|
595
|
-
connect_master_with_toxiproxy!
|
595
|
+
connect_master_with_toxiproxy!
|
596
596
|
|
597
597
|
table_create(:users)
|
598
598
|
100.times { |n| execute("insert into users set reference = '#{ n }'") }
|
@@ -610,7 +610,7 @@ describe Lhm do
|
|
610
610
|
end
|
611
611
|
|
612
612
|
it "should reconnect if reconnect_with_consistent_host is true" do
|
613
|
-
connect_master_with_toxiproxy!
|
613
|
+
connect_master_with_toxiproxy!
|
614
614
|
mysql_disabled = false
|
615
615
|
|
616
616
|
table_create(:users)
|
@@ -635,7 +635,7 @@ describe Lhm do
|
|
635
635
|
method_added(:insert_and_return_count_of_rows_created)
|
636
636
|
end
|
637
637
|
|
638
|
-
Lhm.change_table(:users, :
|
638
|
+
Lhm.change_table(:users, atomic_switch: false, reconnect_with_consistent_host: true) do |t|
|
639
639
|
t.ddl("ALTER TABLE #{t.name} CHANGE id id bigint (20) NOT NULL")
|
640
640
|
t.ddl("ALTER TABLE #{t.name} DROP PRIMARY KEY, ADD PRIMARY KEY (username, id)")
|
641
641
|
t.ddl("ALTER TABLE #{t.name} ADD INDEX (id)")
|
data/spec/unit/chunker_spec.rb
CHANGED
@@ -12,7 +12,8 @@ require 'lhm/connection'
|
|
12
12
|
describe Lhm::Chunker do
|
13
13
|
include UnitHelper
|
14
14
|
|
15
|
-
|
15
|
+
EXPECTED_RETRY_FLAGS_CHUNKER = {:should_retry => true, :log_prefix => "Chunker"}
|
16
|
+
EXPECTED_RETRY_FLAGS_CHUNK_INSERT = {:should_retry => true, :log_prefix => "ChunkInsert"}
|
16
17
|
|
17
18
|
before(:each) do
|
18
19
|
@origin = Lhm::Table.new('foo')
|
@@ -41,11 +42,11 @@ describe Lhm::Chunker do
|
|
41
42
|
5
|
42
43
|
end
|
43
44
|
|
44
|
-
@connection.expects(:select_value).with(regexp_matches(/where id >= 1 order by id limit 1 offset 4/),
|
45
|
-
@connection.expects(:select_value).with(regexp_matches(/where id >= 8 order by id limit 1 offset 4/),
|
46
|
-
@connection.expects(:update).with(regexp_matches(/between 1 and 7/),
|
47
|
-
@connection.expects(:update).with(regexp_matches(/between 8 and 10/),
|
48
|
-
@connection.expects(:execute).twice.with(regexp_matches(/show warnings/),
|
45
|
+
@connection.expects(:select_value).with(regexp_matches(/where id >= 1 order by id limit 1 offset 4/),EXPECTED_RETRY_FLAGS_CHUNKER).returns(7)
|
46
|
+
@connection.expects(:select_value).with(regexp_matches(/where id >= 8 order by id limit 1 offset 4/),EXPECTED_RETRY_FLAGS_CHUNKER).returns(21)
|
47
|
+
@connection.expects(:update).with(regexp_matches(/between 1 and 7/),EXPECTED_RETRY_FLAGS_CHUNK_INSERT).returns(2)
|
48
|
+
@connection.expects(:update).with(regexp_matches(/between 8 and 10/),EXPECTED_RETRY_FLAGS_CHUNK_INSERT).returns(2)
|
49
|
+
@connection.expects(:execute).twice.with(regexp_matches(/show warnings/),EXPECTED_RETRY_FLAGS_CHUNKER).returns([])
|
49
50
|
|
50
51
|
@chunker.run
|
51
52
|
end
|
@@ -56,17 +57,17 @@ describe Lhm::Chunker do
|
|
56
57
|
2
|
57
58
|
end
|
58
59
|
|
59
|
-
@connection.expects(:select_value).with(regexp_matches(/where id >= 1 order by id limit 1 offset 1/),
|
60
|
-
@connection.expects(:select_value).with(regexp_matches(/where id >= 3 order by id limit 1 offset 1/),
|
61
|
-
@connection.expects(:select_value).with(regexp_matches(/where id >= 5 order by id limit 1 offset 1/),
|
62
|
-
@connection.expects(:select_value).with(regexp_matches(/where id >= 7 order by id limit 1 offset 1/),
|
63
|
-
@connection.expects(:select_value).with(regexp_matches(/where id >= 9 order by id limit 1 offset 1/),
|
60
|
+
@connection.expects(:select_value).with(regexp_matches(/where id >= 1 order by id limit 1 offset 1/),EXPECTED_RETRY_FLAGS_CHUNKER).returns(2)
|
61
|
+
@connection.expects(:select_value).with(regexp_matches(/where id >= 3 order by id limit 1 offset 1/),EXPECTED_RETRY_FLAGS_CHUNKER).returns(4)
|
62
|
+
@connection.expects(:select_value).with(regexp_matches(/where id >= 5 order by id limit 1 offset 1/),EXPECTED_RETRY_FLAGS_CHUNKER).returns(6)
|
63
|
+
@connection.expects(:select_value).with(regexp_matches(/where id >= 7 order by id limit 1 offset 1/),EXPECTED_RETRY_FLAGS_CHUNKER).returns(8)
|
64
|
+
@connection.expects(:select_value).with(regexp_matches(/where id >= 9 order by id limit 1 offset 1/),EXPECTED_RETRY_FLAGS_CHUNKER).returns(10)
|
64
65
|
|
65
|
-
@connection.expects(:update).with(regexp_matches(/between 1 and 2/),
|
66
|
-
@connection.expects(:update).with(regexp_matches(/between 3 and 4/),
|
67
|
-
@connection.expects(:update).with(regexp_matches(/between 5 and 6/),
|
68
|
-
@connection.expects(:update).with(regexp_matches(/between 7 and 8/),
|
69
|
-
@connection.expects(:update).with(regexp_matches(/between 9 and 10/),
|
66
|
+
@connection.expects(:update).with(regexp_matches(/between 1 and 2/),EXPECTED_RETRY_FLAGS_CHUNK_INSERT).returns(2)
|
67
|
+
@connection.expects(:update).with(regexp_matches(/between 3 and 4/),EXPECTED_RETRY_FLAGS_CHUNK_INSERT).returns(2)
|
68
|
+
@connection.expects(:update).with(regexp_matches(/between 5 and 6/),EXPECTED_RETRY_FLAGS_CHUNK_INSERT).returns(2)
|
69
|
+
@connection.expects(:update).with(regexp_matches(/between 7 and 8/),EXPECTED_RETRY_FLAGS_CHUNK_INSERT).returns(2)
|
70
|
+
@connection.expects(:update).with(regexp_matches(/between 9 and 10/),EXPECTED_RETRY_FLAGS_CHUNK_INSERT).returns(2)
|
70
71
|
|
71
72
|
@chunker.run
|
72
73
|
end
|
@@ -83,17 +84,17 @@ describe Lhm::Chunker do
|
|
83
84
|
end
|
84
85
|
end
|
85
86
|
|
86
|
-
@connection.expects(:select_value).with(regexp_matches(/where id >= 1 order by id limit 1 offset 1/),
|
87
|
-
@connection.expects(:select_value).with(regexp_matches(/where id >= 3 order by id limit 1 offset 2/),
|
88
|
-
@connection.expects(:select_value).with(regexp_matches(/where id >= 6 order by id limit 1 offset 2/),
|
89
|
-
@connection.expects(:select_value).with(regexp_matches(/where id >= 9 order by id limit 1 offset 2/),
|
87
|
+
@connection.expects(:select_value).with(regexp_matches(/where id >= 1 order by id limit 1 offset 1/),EXPECTED_RETRY_FLAGS_CHUNKER).returns(2)
|
88
|
+
@connection.expects(:select_value).with(regexp_matches(/where id >= 3 order by id limit 1 offset 2/),EXPECTED_RETRY_FLAGS_CHUNKER).returns(5)
|
89
|
+
@connection.expects(:select_value).with(regexp_matches(/where id >= 6 order by id limit 1 offset 2/),EXPECTED_RETRY_FLAGS_CHUNKER).returns(8)
|
90
|
+
@connection.expects(:select_value).with(regexp_matches(/where id >= 9 order by id limit 1 offset 2/),EXPECTED_RETRY_FLAGS_CHUNKER).returns(nil)
|
90
91
|
|
91
|
-
@connection.expects(:update).with(regexp_matches(/between 1 and 2/),
|
92
|
-
@connection.expects(:update).with(regexp_matches(/between 3 and 5/),
|
93
|
-
@connection.expects(:update).with(regexp_matches(/between 6 and 8/),
|
94
|
-
@connection.expects(:update).with(regexp_matches(/between 9 and 10/),
|
92
|
+
@connection.expects(:update).with(regexp_matches(/between 1 and 2/),EXPECTED_RETRY_FLAGS_CHUNK_INSERT).returns(2)
|
93
|
+
@connection.expects(:update).with(regexp_matches(/between 3 and 5/),EXPECTED_RETRY_FLAGS_CHUNK_INSERT).returns(2)
|
94
|
+
@connection.expects(:update).with(regexp_matches(/between 6 and 8/),EXPECTED_RETRY_FLAGS_CHUNK_INSERT).returns(2)
|
95
|
+
@connection.expects(:update).with(regexp_matches(/between 9 and 10/),EXPECTED_RETRY_FLAGS_CHUNK_INSERT).returns(2)
|
95
96
|
|
96
|
-
@connection.expects(:execute).twice.with(regexp_matches(/show warnings/),
|
97
|
+
@connection.expects(:execute).twice.with(regexp_matches(/show warnings/),EXPECTED_RETRY_FLAGS_CHUNKER).returns([])
|
97
98
|
|
98
99
|
@chunker.run
|
99
100
|
end
|
@@ -103,8 +104,8 @@ describe Lhm::Chunker do
|
|
103
104
|
:start => 1,
|
104
105
|
:limit => 1)
|
105
106
|
|
106
|
-
@connection.expects(:select_value).with(regexp_matches(/where id >= 1 order by id limit 1 offset 0/),
|
107
|
-
@connection.expects(:update).with(regexp_matches(/between 1 and 1/),
|
107
|
+
@connection.expects(:select_value).with(regexp_matches(/where id >= 1 order by id limit 1 offset 0/),EXPECTED_RETRY_FLAGS_CHUNKER).returns(nil)
|
108
|
+
@connection.expects(:update).with(regexp_matches(/between 1 and 1/),EXPECTED_RETRY_FLAGS_CHUNK_INSERT).returns(1)
|
108
109
|
|
109
110
|
@chunker.run
|
110
111
|
end
|
@@ -117,17 +118,17 @@ describe Lhm::Chunker do
|
|
117
118
|
2
|
118
119
|
end
|
119
120
|
|
120
|
-
@connection.expects(:select_value).with(regexp_matches(/where id >= 2 order by id limit 1 offset 1/),
|
121
|
-
@connection.expects(:select_value).with(regexp_matches(/where id >= 4 order by id limit 1 offset 1/),
|
122
|
-
@connection.expects(:select_value).with(regexp_matches(/where id >= 6 order by id limit 1 offset 1/),
|
123
|
-
@connection.expects(:select_value).with(regexp_matches(/where id >= 8 order by id limit 1 offset 1/),
|
124
|
-
@connection.expects(:select_value).with(regexp_matches(/where id >= 10 order by id limit 1 offset 1/),
|
121
|
+
@connection.expects(:select_value).with(regexp_matches(/where id >= 2 order by id limit 1 offset 1/),EXPECTED_RETRY_FLAGS_CHUNKER).returns(3)
|
122
|
+
@connection.expects(:select_value).with(regexp_matches(/where id >= 4 order by id limit 1 offset 1/),EXPECTED_RETRY_FLAGS_CHUNKER).returns(5)
|
123
|
+
@connection.expects(:select_value).with(regexp_matches(/where id >= 6 order by id limit 1 offset 1/),EXPECTED_RETRY_FLAGS_CHUNKER).returns(7)
|
124
|
+
@connection.expects(:select_value).with(regexp_matches(/where id >= 8 order by id limit 1 offset 1/),EXPECTED_RETRY_FLAGS_CHUNKER).returns(9)
|
125
|
+
@connection.expects(:select_value).with(regexp_matches(/where id >= 10 order by id limit 1 offset 1/),EXPECTED_RETRY_FLAGS_CHUNKER).returns(nil)
|
125
126
|
|
126
|
-
@connection.expects(:update).with(regexp_matches(/between 2 and 3/),
|
127
|
-
@connection.expects(:update).with(regexp_matches(/between 4 and 5/),
|
128
|
-
@connection.expects(:update).with(regexp_matches(/between 6 and 7/),
|
129
|
-
@connection.expects(:update).with(regexp_matches(/between 8 and 9/),
|
130
|
-
@connection.expects(:update).with(regexp_matches(/between 10 and 10/),
|
127
|
+
@connection.expects(:update).with(regexp_matches(/between 2 and 3/),EXPECTED_RETRY_FLAGS_CHUNK_INSERT).returns(2)
|
128
|
+
@connection.expects(:update).with(regexp_matches(/between 4 and 5/),EXPECTED_RETRY_FLAGS_CHUNK_INSERT).returns(2)
|
129
|
+
@connection.expects(:update).with(regexp_matches(/between 6 and 7/),EXPECTED_RETRY_FLAGS_CHUNK_INSERT).returns(2)
|
130
|
+
@connection.expects(:update).with(regexp_matches(/between 8 and 9/),EXPECTED_RETRY_FLAGS_CHUNK_INSERT).returns(2)
|
131
|
+
@connection.expects(:update).with(regexp_matches(/between 10 and 10/),EXPECTED_RETRY_FLAGS_CHUNK_INSERT).returns(1)
|
131
132
|
|
132
133
|
@chunker.run
|
133
134
|
end
|
@@ -141,9 +142,9 @@ describe Lhm::Chunker do
|
|
141
142
|
2
|
142
143
|
end
|
143
144
|
|
144
|
-
@connection.expects(:select_value).with(regexp_matches(/where id >= 1 order by id limit 1 offset 1/),
|
145
|
-
@connection.expects(:update).with(regexp_matches(/where \(foo.created_at > '2013-07-10' or foo.baz = 'quux'\) and `foo`/),
|
146
|
-
@connection.expects(:execute).with(regexp_matches(/show warnings/),
|
145
|
+
@connection.expects(:select_value).with(regexp_matches(/where id >= 1 order by id limit 1 offset 1/),EXPECTED_RETRY_FLAGS_CHUNKER).returns(2)
|
146
|
+
@connection.expects(:update).with(regexp_matches(/where \(foo.created_at > '2013-07-10' or foo.baz = 'quux'\) and `foo`/),EXPECTED_RETRY_FLAGS_CHUNK_INSERT).returns(1)
|
147
|
+
@connection.expects(:execute).with(regexp_matches(/show warnings/),EXPECTED_RETRY_FLAGS_CHUNKER).returns([])
|
147
148
|
|
148
149
|
def @migration.conditions
|
149
150
|
"where foo.created_at > '2013-07-10' or foo.baz = 'quux'"
|
@@ -161,9 +162,9 @@ describe Lhm::Chunker do
|
|
161
162
|
2
|
162
163
|
end
|
163
164
|
|
164
|
-
@connection.expects(:select_value).with(regexp_matches(/where id >= 1 order by id limit 1 offset 1/),
|
165
|
-
@connection.expects(:update).with(regexp_matches(/inner join bar on foo.id = bar.foo_id and/),
|
166
|
-
@connection.expects(:execute).with(regexp_matches(/show warnings/),
|
165
|
+
@connection.expects(:select_value).with(regexp_matches(/where id >= 1 order by id limit 1 offset 1/),EXPECTED_RETRY_FLAGS_CHUNKER).returns(2)
|
166
|
+
@connection.expects(:update).with(regexp_matches(/inner join bar on foo.id = bar.foo_id and/),EXPECTED_RETRY_FLAGS_CHUNK_INSERT).returns(1)
|
167
|
+
@connection.expects(:execute).with(regexp_matches(/show warnings/),EXPECTED_RETRY_FLAGS_CHUNKER).returns([])
|
167
168
|
|
168
169
|
def @migration.conditions
|
169
170
|
'inner join bar on foo.id = bar.foo_id'
|
@@ -15,9 +15,13 @@ describe Lhm::Connection do
|
|
15
15
|
ar_connection.stubs(:execute).raises(LOCK_WAIT).then.returns(true)
|
16
16
|
ar_connection.stubs(:active?).returns(true)
|
17
17
|
|
18
|
-
connection = Lhm::Connection.new(connection: ar_connection
|
18
|
+
connection = Lhm::Connection.new(connection: ar_connection, options: {
|
19
|
+
retriable: {
|
20
|
+
base_interval: 0
|
21
|
+
}
|
22
|
+
})
|
19
23
|
|
20
|
-
connection.execute("SHOW TABLES", should_retry: true
|
24
|
+
connection.execute("SHOW TABLES", should_retry: true)
|
21
25
|
|
22
26
|
log_messages = @logs.string.split("\n")
|
23
27
|
assert_equal(1, log_messages.length)
|
@@ -31,9 +35,14 @@ describe Lhm::Connection do
|
|
31
35
|
.then.returns(true)
|
32
36
|
ar_connection.stubs(:active?).returns(true)
|
33
37
|
|
34
|
-
connection = Lhm::Connection.new(connection: ar_connection
|
38
|
+
connection = Lhm::Connection.new(connection: ar_connection, options: {
|
39
|
+
retriable: {
|
40
|
+
base_interval: 0,
|
41
|
+
tries: 3
|
42
|
+
}
|
43
|
+
})
|
35
44
|
|
36
|
-
connection.execute("SHOW TABLES", should_retry: true
|
45
|
+
connection.execute("SHOW TABLES", should_retry: true)
|
37
46
|
|
38
47
|
log_messages = @logs.string.split("\n")
|
39
48
|
assert_equal(2, log_messages.length)
|
@@ -46,9 +55,14 @@ describe Lhm::Connection do
|
|
46
55
|
.then.returns(1)
|
47
56
|
ar_connection.stubs(:active?).returns(true)
|
48
57
|
|
49
|
-
connection = Lhm::Connection.new(connection: ar_connection
|
58
|
+
connection = Lhm::Connection.new(connection: ar_connection, options: {
|
59
|
+
retriable: {
|
60
|
+
base_interval: 0,
|
61
|
+
tries: 3
|
62
|
+
}
|
63
|
+
})
|
50
64
|
|
51
|
-
val = connection.update("SHOW TABLES", should_retry: true
|
65
|
+
val = connection.update("SHOW TABLES", should_retry: true)
|
52
66
|
|
53
67
|
log_messages = @logs.string.split("\n")
|
54
68
|
assert_equal val, 1
|
@@ -62,24 +76,35 @@ describe Lhm::Connection do
|
|
62
76
|
.then.returns("dummy")
|
63
77
|
ar_connection.stubs(:active?).returns(true)
|
64
78
|
|
65
|
-
connection = Lhm::Connection.new(connection: ar_connection
|
79
|
+
connection = Lhm::Connection.new(connection: ar_connection, options: {
|
80
|
+
retriable: {
|
81
|
+
base_interval: 0,
|
82
|
+
tries: 3
|
83
|
+
}
|
84
|
+
})
|
66
85
|
|
67
|
-
val = connection.select_value("SHOW TABLES", should_retry: true
|
86
|
+
val = connection.select_value("SHOW TABLES", should_retry: true)
|
68
87
|
|
69
88
|
log_messages = @logs.string.split("\n")
|
70
89
|
assert_equal val, "dummy"
|
71
90
|
assert_equal(2, log_messages.length)
|
72
91
|
end
|
73
92
|
|
74
|
-
it "Queries should be tagged with ProxySQL tag if
|
93
|
+
it "Queries should be tagged with ProxySQL tag if reconnect_with_consistent_host is enabled" do
|
75
94
|
ar_connection = mock()
|
76
95
|
ar_connection.expects(:public_send).with(:select_value, "SHOW TABLES #{Lhm::ProxySQLHelper::ANNOTATION}").returns("dummy")
|
77
96
|
ar_connection.stubs(:execute).times(4).returns([["dummy"]])
|
78
97
|
ar_connection.stubs(:active?).returns(true)
|
79
98
|
|
80
|
-
connection = Lhm::Connection.new(connection: ar_connection, options: {
|
99
|
+
connection = Lhm::Connection.new(connection: ar_connection, options: {
|
100
|
+
reconnect_with_consistent_host: true,
|
101
|
+
retriable: {
|
102
|
+
base_interval: 0,
|
103
|
+
tries: 3
|
104
|
+
}
|
105
|
+
})
|
81
106
|
|
82
|
-
val = connection.select_value("SHOW TABLES", should_retry: true
|
107
|
+
val = connection.select_value("SHOW TABLES", should_retry: true)
|
83
108
|
|
84
109
|
assert_equal val, "dummy"
|
85
110
|
end
|
data/spec/unit/entangler_spec.rb
CHANGED
@@ -69,9 +69,15 @@ describe Lhm::Entangler do
|
|
69
69
|
.raises(Mysql2::Error, 'Lock wait timeout exceeded; try restarting transaction')
|
70
70
|
ar_connection.stubs(:active?).returns(true)
|
71
71
|
|
72
|
-
connection = Lhm::Connection.new(connection: ar_connection, options: {
|
72
|
+
connection = Lhm::Connection.new(connection: ar_connection, options: {
|
73
|
+
reconnect_with_consistent_host: true,
|
74
|
+
retriable: {
|
75
|
+
base_interval: 0,
|
76
|
+
tries: tries
|
77
|
+
}
|
78
|
+
})
|
73
79
|
|
74
|
-
@entangler = Lhm::Entangler.new(@migration, connection
|
80
|
+
@entangler = Lhm::Entangler.new(@migration, connection)
|
75
81
|
|
76
82
|
assert_raises(Mysql2::Error) { @entangler.before }
|
77
83
|
end
|
@@ -83,9 +89,14 @@ describe Lhm::Entangler do
|
|
83
89
|
.then
|
84
90
|
.raises(Mysql2::Error, 'The MySQL server is running with the --read-only option so it cannot execute this statement.')
|
85
91
|
ar_connection.stubs(:active?).returns(true)
|
86
|
-
connection = Lhm::Connection.new(connection: ar_connection, options: {
|
87
|
-
|
88
|
-
|
92
|
+
connection = Lhm::Connection.new(connection: ar_connection, options: {
|
93
|
+
reconnect_with_consistent_host: true,
|
94
|
+
retriable: {
|
95
|
+
base_interval: 0
|
96
|
+
},
|
97
|
+
})
|
98
|
+
|
99
|
+
@entangler = Lhm::Entangler.new(@migration, connection)
|
89
100
|
assert_raises(Mysql2::Error) { @entangler.before }
|
90
101
|
end
|
91
102
|
|
@@ -99,9 +110,14 @@ describe Lhm::Entangler do
|
|
99
110
|
.returns([["dummy"]])
|
100
111
|
ar_connection.stubs(:active?).returns(true)
|
101
112
|
|
102
|
-
connection = Lhm::Connection.new(connection: ar_connection, options: {
|
113
|
+
connection = Lhm::Connection.new(connection: ar_connection, options: {
|
114
|
+
reconnect_with_consistent_host: true,
|
115
|
+
retriable: {
|
116
|
+
base_interval: 0
|
117
|
+
},
|
118
|
+
})
|
103
119
|
|
104
|
-
@entangler = Lhm::Entangler.new(@migration, connection
|
120
|
+
@entangler = Lhm::Entangler.new(@migration, connection)
|
105
121
|
|
106
122
|
assert @entangler.before
|
107
123
|
end
|
@@ -126,9 +142,15 @@ describe Lhm::Entangler do
|
|
126
142
|
.raises(Mysql2::Error, 'Lock wait timeout exceeded; try restarting transaction') # final error
|
127
143
|
ar_connection.stubs(:active?).returns(true)
|
128
144
|
|
129
|
-
connection = Lhm::Connection.new(connection: ar_connection, options: {
|
145
|
+
connection = Lhm::Connection.new(connection: ar_connection, options: {
|
146
|
+
reconnect_with_consistent_host: true,
|
147
|
+
retriable: {
|
148
|
+
tries: 5,
|
149
|
+
base_interval: 0
|
150
|
+
},
|
151
|
+
})
|
130
152
|
|
131
|
-
@entangler = Lhm::Entangler.new(@migration, connection
|
153
|
+
@entangler = Lhm::Entangler.new(@migration, connection)
|
132
154
|
|
133
155
|
assert_raises(Mysql2::Error) { @entangler.before }
|
134
156
|
end
|
@@ -142,7 +142,7 @@ describe Lhm::Throttler::Slave do
|
|
142
142
|
Lhm::Throttler::Slave.any_instance.stubs(:config).returns([])
|
143
143
|
|
144
144
|
slave = Lhm::Throttler::Slave.new('slave', @dummy_mysql_client_config)
|
145
|
-
|
145
|
+
Logger.any_instance.expects(:info).with("Unable to connect and/or query slave: Can't connect to MySQL server")
|
146
146
|
assert_equal(0, slave.lag)
|
147
147
|
end
|
148
148
|
end
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: lhm-shopify
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 3.5.
|
4
|
+
version: 3.5.4
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- SoundCloud
|
@@ -12,7 +12,7 @@ authors:
|
|
12
12
|
autorequire:
|
13
13
|
bindir: bin
|
14
14
|
cert_chain: []
|
15
|
-
date: 2021-12-
|
15
|
+
date: 2021-12-14 00:00:00.000000000 Z
|
16
16
|
dependencies:
|
17
17
|
- !ruby/object:Gem::Dependency
|
18
18
|
name: retriable
|