lhm-teak 3.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.github/workflows/test.yml +43 -0
- data/.gitignore +12 -0
- data/.rubocop.yml +183 -0
- data/.travis.yml +21 -0
- data/Appraisals +24 -0
- data/CHANGELOG.md +254 -0
- data/Gemfile +5 -0
- data/Gemfile.lock +67 -0
- data/LICENSE +27 -0
- data/README.md +335 -0
- data/Rakefile +33 -0
- data/dev.yml +45 -0
- data/docker-compose.yml +60 -0
- data/gemfiles/activerecord_5.2.gemfile +9 -0
- data/gemfiles/activerecord_5.2.gemfile.lock +66 -0
- data/gemfiles/activerecord_6.0.gemfile +7 -0
- data/gemfiles/activerecord_6.0.gemfile.lock +68 -0
- data/gemfiles/activerecord_6.1.gemfile +7 -0
- data/gemfiles/activerecord_6.1.gemfile.lock +67 -0
- data/gemfiles/activerecord_7.0.0.alpha2.gemfile +7 -0
- data/gemfiles/activerecord_7.0.0.alpha2.gemfile.lock +65 -0
- data/lhm.gemspec +38 -0
- data/lib/lhm/atomic_switcher.rb +46 -0
- data/lib/lhm/chunk_finder.rb +62 -0
- data/lib/lhm/chunk_insert.rb +61 -0
- data/lib/lhm/chunker.rb +95 -0
- data/lib/lhm/cleanup/current.rb +71 -0
- data/lib/lhm/command.rb +48 -0
- data/lib/lhm/connection.rb +108 -0
- data/lib/lhm/entangler.rb +112 -0
- data/lib/lhm/intersection.rb +51 -0
- data/lib/lhm/invoker.rb +100 -0
- data/lib/lhm/locked_switcher.rb +76 -0
- data/lib/lhm/migration.rb +51 -0
- data/lib/lhm/migrator.rb +244 -0
- data/lib/lhm/printer.rb +63 -0
- data/lib/lhm/proxysql_helper.rb +10 -0
- data/lib/lhm/railtie.rb +9 -0
- data/lib/lhm/sql_helper.rb +77 -0
- data/lib/lhm/sql_retry.rb +180 -0
- data/lib/lhm/table.rb +121 -0
- data/lib/lhm/table_name.rb +23 -0
- data/lib/lhm/test_support.rb +35 -0
- data/lib/lhm/throttler/slave_lag.rb +162 -0
- data/lib/lhm/throttler/threads_running.rb +53 -0
- data/lib/lhm/throttler/time.rb +29 -0
- data/lib/lhm/throttler.rb +36 -0
- data/lib/lhm/timestamp.rb +11 -0
- data/lib/lhm/version.rb +6 -0
- data/lib/lhm-shopify.rb +1 -0
- data/lib/lhm.rb +156 -0
- data/scripts/helpers/wait-for-dbs.sh +21 -0
- data/scripts/mysql/reader/create_replication.sql +10 -0
- data/scripts/mysql/writer/create_test_db.sql +1 -0
- data/scripts/mysql/writer/create_users.sql +6 -0
- data/scripts/proxysql/proxysql.cnf +117 -0
- data/shipit.rubygems.yml +0 -0
- data/spec/.lhm.example +4 -0
- data/spec/README.md +58 -0
- data/spec/fixtures/bigint_table.ddl +4 -0
- data/spec/fixtures/composite_primary_key.ddl +6 -0
- data/spec/fixtures/composite_primary_key_dest.ddl +6 -0
- data/spec/fixtures/custom_primary_key.ddl +6 -0
- data/spec/fixtures/custom_primary_key_dest.ddl +6 -0
- data/spec/fixtures/destination.ddl +6 -0
- data/spec/fixtures/lines.ddl +7 -0
- data/spec/fixtures/origin.ddl +6 -0
- data/spec/fixtures/permissions.ddl +5 -0
- data/spec/fixtures/small_table.ddl +4 -0
- data/spec/fixtures/tracks.ddl +5 -0
- data/spec/fixtures/users.ddl +14 -0
- data/spec/fixtures/wo_id_int_column.ddl +6 -0
- data/spec/integration/atomic_switcher_spec.rb +129 -0
- data/spec/integration/chunk_insert_spec.rb +30 -0
- data/spec/integration/chunker_spec.rb +269 -0
- data/spec/integration/cleanup_spec.rb +147 -0
- data/spec/integration/database.yml +25 -0
- data/spec/integration/entangler_spec.rb +68 -0
- data/spec/integration/integration_helper.rb +252 -0
- data/spec/integration/invoker_spec.rb +33 -0
- data/spec/integration/lhm_spec.rb +659 -0
- data/spec/integration/lock_wait_timeout_spec.rb +30 -0
- data/spec/integration/locked_switcher_spec.rb +50 -0
- data/spec/integration/proxysql_spec.rb +34 -0
- data/spec/integration/sql_retry/db_connection_helper.rb +52 -0
- data/spec/integration/sql_retry/lock_wait_spec.rb +127 -0
- data/spec/integration/sql_retry/lock_wait_timeout_test_helper.rb +114 -0
- data/spec/integration/sql_retry/proxysql_helper.rb +22 -0
- data/spec/integration/sql_retry/retry_with_proxysql_spec.rb +109 -0
- data/spec/integration/table_spec.rb +83 -0
- data/spec/integration/toxiproxy_helper.rb +40 -0
- data/spec/test_helper.rb +69 -0
- data/spec/unit/atomic_switcher_spec.rb +29 -0
- data/spec/unit/chunk_finder_spec.rb +73 -0
- data/spec/unit/chunk_insert_spec.rb +67 -0
- data/spec/unit/chunker_spec.rb +176 -0
- data/spec/unit/connection_spec.rb +111 -0
- data/spec/unit/entangler_spec.rb +187 -0
- data/spec/unit/intersection_spec.rb +51 -0
- data/spec/unit/lhm_spec.rb +46 -0
- data/spec/unit/locked_switcher_spec.rb +46 -0
- data/spec/unit/migrator_spec.rb +144 -0
- data/spec/unit/printer_spec.rb +85 -0
- data/spec/unit/sql_helper_spec.rb +28 -0
- data/spec/unit/table_name_spec.rb +39 -0
- data/spec/unit/table_spec.rb +47 -0
- data/spec/unit/throttler/slave_lag_spec.rb +322 -0
- data/spec/unit/throttler/threads_running_spec.rb +64 -0
- data/spec/unit/throttler_spec.rb +124 -0
- data/spec/unit/unit_helper.rb +26 -0
- metadata +366 -0
|
@@ -0,0 +1,252 @@
|
|
|
1
|
+
# Copyright (c) 2011 - 2013, SoundCloud Ltd., Rany Keddo, Tobias Bielohlawek, Tobias
|
|
2
|
+
# Schmidt
|
|
3
|
+
require 'test_helper'
|
|
4
|
+
require 'yaml'
|
|
5
|
+
require 'active_support'
|
|
6
|
+
require 'logger'
|
|
7
|
+
|
|
8
|
+
begin
|
|
9
|
+
$db_config = YAML.load_file(File.expand_path(File.dirname(__FILE__)) + '/database.yml')
|
|
10
|
+
rescue StandardError => e
|
|
11
|
+
puts "Run install.sh to setup database"
|
|
12
|
+
raise e
|
|
13
|
+
end
|
|
14
|
+
|
|
15
|
+
$db_name = 'test'
|
|
16
|
+
|
|
17
|
+
require 'lhm/table'
|
|
18
|
+
require 'lhm/sql_helper'
|
|
19
|
+
|
|
20
|
+
module IntegrationHelper
|
|
21
|
+
|
|
22
|
+
def self.included(base)
|
|
23
|
+
base.after(:each) do
|
|
24
|
+
cleanup_connection = new_mysql_connection
|
|
25
|
+
results = cleanup_connection.query("SELECT table_name FROM information_schema.tables WHERE table_schema = '#{$db_name}';")
|
|
26
|
+
table_names_for_cleanup = results.map { |row| "#{$db_name}." + row.values.first }
|
|
27
|
+
cleanup_connection.query("DROP TABLE IF EXISTS #{table_names_for_cleanup.join(', ')};") if table_names_for_cleanup.length > 0
|
|
28
|
+
end
|
|
29
|
+
end
|
|
30
|
+
|
|
31
|
+
#
|
|
32
|
+
# Connectivity
|
|
33
|
+
#
|
|
34
|
+
def connection
|
|
35
|
+
@connection
|
|
36
|
+
end
|
|
37
|
+
|
|
38
|
+
def connect_proxysql!
|
|
39
|
+
connect!(
|
|
40
|
+
'127.0.0.1',
|
|
41
|
+
$db_config['proxysql']['port'],
|
|
42
|
+
$db_config['proxysql']['user'],
|
|
43
|
+
$db_config['proxysql']['password'],
|
|
44
|
+
)
|
|
45
|
+
end
|
|
46
|
+
|
|
47
|
+
def connect_master!
|
|
48
|
+
connect!(
|
|
49
|
+
'127.0.0.1',
|
|
50
|
+
$db_config['master']['port'],
|
|
51
|
+
$db_config['master']['user'],
|
|
52
|
+
$db_config['master']['password'],
|
|
53
|
+
)
|
|
54
|
+
end
|
|
55
|
+
|
|
56
|
+
def connect_slave!
|
|
57
|
+
connect!(
|
|
58
|
+
'127.0.0.1',
|
|
59
|
+
$db_config['slave']['port'],
|
|
60
|
+
$db_config['slave']['user'],
|
|
61
|
+
$db_config['slave']['password'],
|
|
62
|
+
)
|
|
63
|
+
end
|
|
64
|
+
|
|
65
|
+
def connect_master_with_toxiproxy!
|
|
66
|
+
connect!(
|
|
67
|
+
'127.0.0.1',
|
|
68
|
+
$db_config['master_toxic']['port'],
|
|
69
|
+
$db_config['master_toxic']['user'],
|
|
70
|
+
$db_config['master_toxic']['password'])
|
|
71
|
+
end
|
|
72
|
+
|
|
73
|
+
def connect!(hostname, port, user, password)
|
|
74
|
+
Lhm.setup(ar_conn(hostname, port, user, password))
|
|
75
|
+
unless defined?(@@cleaned_up)
|
|
76
|
+
Lhm.cleanup(true)
|
|
77
|
+
@@cleaned_up = true
|
|
78
|
+
end
|
|
79
|
+
@connection = Lhm.connection
|
|
80
|
+
end
|
|
81
|
+
|
|
82
|
+
def ar_conn(host, port, user, password)
|
|
83
|
+
ActiveRecord::Base.establish_connection(
|
|
84
|
+
:adapter => 'mysql2',
|
|
85
|
+
:host => host,
|
|
86
|
+
:username => user,
|
|
87
|
+
:port => port,
|
|
88
|
+
:password => password,
|
|
89
|
+
:database => $db_name
|
|
90
|
+
)
|
|
91
|
+
ActiveRecord::Base.connection
|
|
92
|
+
end
|
|
93
|
+
|
|
94
|
+
def select_one(*args)
|
|
95
|
+
@connection.select_one(*args)
|
|
96
|
+
end
|
|
97
|
+
|
|
98
|
+
def select_value(*args)
|
|
99
|
+
@connection.select_value(*args)
|
|
100
|
+
end
|
|
101
|
+
|
|
102
|
+
def execute(*args)
|
|
103
|
+
retries = 10
|
|
104
|
+
begin
|
|
105
|
+
@connection.execute(*args)
|
|
106
|
+
rescue => e
|
|
107
|
+
if (retries -= 1) > 0 && e.message =~ /Table '.*?' doesn't exist/
|
|
108
|
+
sleep 0.1
|
|
109
|
+
retry
|
|
110
|
+
else
|
|
111
|
+
raise
|
|
112
|
+
end
|
|
113
|
+
end
|
|
114
|
+
end
|
|
115
|
+
|
|
116
|
+
def slave(&block)
|
|
117
|
+
if master_slave_mode?
|
|
118
|
+
connect_slave!
|
|
119
|
+
|
|
120
|
+
# need to wait for the slave to catch up. a better method would be to
|
|
121
|
+
# check the master binlog position and wait for the slave to catch up
|
|
122
|
+
# to that position.
|
|
123
|
+
sleep 1
|
|
124
|
+
else
|
|
125
|
+
connect_master!
|
|
126
|
+
end
|
|
127
|
+
|
|
128
|
+
yield block
|
|
129
|
+
|
|
130
|
+
if master_slave_mode?
|
|
131
|
+
connect_master!
|
|
132
|
+
end
|
|
133
|
+
end
|
|
134
|
+
|
|
135
|
+
# Helps testing behaviour when another client locks the db
|
|
136
|
+
def start_locking_thread(lock_for, queue, locking_query)
|
|
137
|
+
Thread.new do
|
|
138
|
+
conn = new_mysql_connection
|
|
139
|
+
conn.query('BEGIN')
|
|
140
|
+
conn.query(locking_query)
|
|
141
|
+
queue.push(true)
|
|
142
|
+
sleep(lock_for) # Sleep for log so LHM gives up
|
|
143
|
+
conn.query('ROLLBACK')
|
|
144
|
+
end
|
|
145
|
+
end
|
|
146
|
+
|
|
147
|
+
#
|
|
148
|
+
# Test Data
|
|
149
|
+
#
|
|
150
|
+
|
|
151
|
+
def fixture(name)
|
|
152
|
+
File.read($fixtures.join("#{ name }.ddl"))
|
|
153
|
+
end
|
|
154
|
+
|
|
155
|
+
def table_create(fixture_name)
|
|
156
|
+
execute "drop table if exists `#{ fixture_name }`"
|
|
157
|
+
execute fixture(fixture_name)
|
|
158
|
+
table_read(fixture_name)
|
|
159
|
+
end
|
|
160
|
+
|
|
161
|
+
def table_rename(from_name, to_name)
|
|
162
|
+
execute "rename table `#{ from_name }` to `#{ to_name }`"
|
|
163
|
+
end
|
|
164
|
+
|
|
165
|
+
def table_read(fixture_name)
|
|
166
|
+
Lhm::Table.parse(fixture_name, @connection)
|
|
167
|
+
end
|
|
168
|
+
|
|
169
|
+
def data_source_exists?(table)
|
|
170
|
+
connection.data_source_exists?(table.name)
|
|
171
|
+
end
|
|
172
|
+
|
|
173
|
+
def new_mysql_connection(role='master')
|
|
174
|
+
Mysql2::Client.new(
|
|
175
|
+
host: '127.0.0.1',
|
|
176
|
+
database: $db_name,
|
|
177
|
+
username: $db_config[role]['user'],
|
|
178
|
+
password: $db_config[role]['password'],
|
|
179
|
+
port: $db_config[role]['port'],
|
|
180
|
+
socket: $db_config[role]['socket']
|
|
181
|
+
)
|
|
182
|
+
end
|
|
183
|
+
|
|
184
|
+
#
|
|
185
|
+
# Database Helpers
|
|
186
|
+
#
|
|
187
|
+
|
|
188
|
+
def count(table, column, value)
|
|
189
|
+
query = "select count(*) from #{ table } where #{ column } = '#{ value }'"
|
|
190
|
+
select_value(query).to_i
|
|
191
|
+
end
|
|
192
|
+
|
|
193
|
+
def count_all(table)
|
|
194
|
+
query = "select count(*) from `#{ table }`"
|
|
195
|
+
select_value(query).to_i
|
|
196
|
+
end
|
|
197
|
+
|
|
198
|
+
def index_on_columns?(table_name, cols, type = :non_unique)
|
|
199
|
+
key_name = Lhm::SqlHelper.idx_name(table_name, cols)
|
|
200
|
+
|
|
201
|
+
index?(table_name, key_name, type)
|
|
202
|
+
end
|
|
203
|
+
|
|
204
|
+
def index?(table_name, key_name, type = :non_unique)
|
|
205
|
+
non_unique = type == :non_unique ? 1 : 0
|
|
206
|
+
|
|
207
|
+
!!select_one(%Q<
|
|
208
|
+
show indexes in `#{ table_name }`
|
|
209
|
+
where key_name = '#{ key_name }'
|
|
210
|
+
and non_unique = #{ non_unique }
|
|
211
|
+
>)
|
|
212
|
+
end
|
|
213
|
+
|
|
214
|
+
#
|
|
215
|
+
# Environment
|
|
216
|
+
#
|
|
217
|
+
|
|
218
|
+
def master_slave_mode?
|
|
219
|
+
!!ENV['MASTER_SLAVE']
|
|
220
|
+
end
|
|
221
|
+
|
|
222
|
+
#
|
|
223
|
+
# Misc
|
|
224
|
+
#
|
|
225
|
+
|
|
226
|
+
def capture_stdout
|
|
227
|
+
out = StringIO.new
|
|
228
|
+
$stdout = out
|
|
229
|
+
logger = Logger.new($stdout)
|
|
230
|
+
yield logger
|
|
231
|
+
return out.string
|
|
232
|
+
ensure
|
|
233
|
+
$stdout = ::STDOUT
|
|
234
|
+
end
|
|
235
|
+
|
|
236
|
+
def simulate_failed_migration
|
|
237
|
+
Lhm::Entangler.class_eval do
|
|
238
|
+
alias_method :old_after, :after
|
|
239
|
+
def after
|
|
240
|
+
true
|
|
241
|
+
end
|
|
242
|
+
end
|
|
243
|
+
|
|
244
|
+
yield
|
|
245
|
+
ensure
|
|
246
|
+
Lhm::Entangler.class_eval do
|
|
247
|
+
undef_method :after
|
|
248
|
+
alias_method :after, :old_after
|
|
249
|
+
undef_method :old_after
|
|
250
|
+
end
|
|
251
|
+
end
|
|
252
|
+
end
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
require File.expand_path(File.dirname(__FILE__)) + '/integration_helper'
|
|
2
|
+
|
|
3
|
+
require 'lhm/invoker'
|
|
4
|
+
|
|
5
|
+
describe Lhm::Invoker do
|
|
6
|
+
include IntegrationHelper
|
|
7
|
+
|
|
8
|
+
before(:each) do
|
|
9
|
+
connect_master!
|
|
10
|
+
@origin = table_create('users')
|
|
11
|
+
@destination = table_create('destination')
|
|
12
|
+
@invoker = Lhm::Invoker.new(Lhm::Table.parse(:users, @connection), @connection)
|
|
13
|
+
@migration = Lhm::Migration.new(@origin, @destination)
|
|
14
|
+
@entangler = Lhm::Entangler.new(@migration, @connection)
|
|
15
|
+
@entangler.before
|
|
16
|
+
end
|
|
17
|
+
|
|
18
|
+
after(:each) do
|
|
19
|
+
@entangler.after if @invoker.triggers_still_exist?(@invoker.connection, @entangler)
|
|
20
|
+
end
|
|
21
|
+
|
|
22
|
+
describe 'triggers_still_exist?' do
|
|
23
|
+
it 'should return true when triggers still exist' do
|
|
24
|
+
assert @invoker.triggers_still_exist?(@invoker.connection, @entangler)
|
|
25
|
+
end
|
|
26
|
+
|
|
27
|
+
it 'should return false when triggers do not exist' do
|
|
28
|
+
@entangler.after
|
|
29
|
+
|
|
30
|
+
refute @invoker.triggers_still_exist?(@invoker.connection, @entangler)
|
|
31
|
+
end
|
|
32
|
+
end
|
|
33
|
+
end
|