lhm-teak 3.6.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +7 -0
- data/.github/workflows/test.yml +43 -0
- data/.gitignore +12 -0
- data/.rubocop.yml +183 -0
- data/.travis.yml +21 -0
- data/Appraisals +24 -0
- data/CHANGELOG.md +254 -0
- data/Gemfile +5 -0
- data/Gemfile.lock +67 -0
- data/LICENSE +27 -0
- data/README.md +335 -0
- data/Rakefile +33 -0
- data/dev.yml +45 -0
- data/docker-compose.yml +60 -0
- data/gemfiles/activerecord_5.2.gemfile +9 -0
- data/gemfiles/activerecord_5.2.gemfile.lock +66 -0
- data/gemfiles/activerecord_6.0.gemfile +7 -0
- data/gemfiles/activerecord_6.0.gemfile.lock +68 -0
- data/gemfiles/activerecord_6.1.gemfile +7 -0
- data/gemfiles/activerecord_6.1.gemfile.lock +67 -0
- data/gemfiles/activerecord_7.0.0.alpha2.gemfile +7 -0
- data/gemfiles/activerecord_7.0.0.alpha2.gemfile.lock +65 -0
- data/lhm.gemspec +38 -0
- data/lib/lhm/atomic_switcher.rb +46 -0
- data/lib/lhm/chunk_finder.rb +62 -0
- data/lib/lhm/chunk_insert.rb +61 -0
- data/lib/lhm/chunker.rb +95 -0
- data/lib/lhm/cleanup/current.rb +71 -0
- data/lib/lhm/command.rb +48 -0
- data/lib/lhm/connection.rb +108 -0
- data/lib/lhm/entangler.rb +112 -0
- data/lib/lhm/intersection.rb +51 -0
- data/lib/lhm/invoker.rb +100 -0
- data/lib/lhm/locked_switcher.rb +76 -0
- data/lib/lhm/migration.rb +51 -0
- data/lib/lhm/migrator.rb +244 -0
- data/lib/lhm/printer.rb +63 -0
- data/lib/lhm/proxysql_helper.rb +10 -0
- data/lib/lhm/railtie.rb +9 -0
- data/lib/lhm/sql_helper.rb +77 -0
- data/lib/lhm/sql_retry.rb +180 -0
- data/lib/lhm/table.rb +121 -0
- data/lib/lhm/table_name.rb +23 -0
- data/lib/lhm/test_support.rb +35 -0
- data/lib/lhm/throttler/slave_lag.rb +162 -0
- data/lib/lhm/throttler/threads_running.rb +53 -0
- data/lib/lhm/throttler/time.rb +29 -0
- data/lib/lhm/throttler.rb +36 -0
- data/lib/lhm/timestamp.rb +11 -0
- data/lib/lhm/version.rb +6 -0
- data/lib/lhm-shopify.rb +1 -0
- data/lib/lhm.rb +156 -0
- data/scripts/helpers/wait-for-dbs.sh +21 -0
- data/scripts/mysql/reader/create_replication.sql +10 -0
- data/scripts/mysql/writer/create_test_db.sql +1 -0
- data/scripts/mysql/writer/create_users.sql +6 -0
- data/scripts/proxysql/proxysql.cnf +117 -0
- data/shipit.rubygems.yml +0 -0
- data/spec/.lhm.example +4 -0
- data/spec/README.md +58 -0
- data/spec/fixtures/bigint_table.ddl +4 -0
- data/spec/fixtures/composite_primary_key.ddl +6 -0
- data/spec/fixtures/composite_primary_key_dest.ddl +6 -0
- data/spec/fixtures/custom_primary_key.ddl +6 -0
- data/spec/fixtures/custom_primary_key_dest.ddl +6 -0
- data/spec/fixtures/destination.ddl +6 -0
- data/spec/fixtures/lines.ddl +7 -0
- data/spec/fixtures/origin.ddl +6 -0
- data/spec/fixtures/permissions.ddl +5 -0
- data/spec/fixtures/small_table.ddl +4 -0
- data/spec/fixtures/tracks.ddl +5 -0
- data/spec/fixtures/users.ddl +14 -0
- data/spec/fixtures/wo_id_int_column.ddl +6 -0
- data/spec/integration/atomic_switcher_spec.rb +129 -0
- data/spec/integration/chunk_insert_spec.rb +30 -0
- data/spec/integration/chunker_spec.rb +269 -0
- data/spec/integration/cleanup_spec.rb +147 -0
- data/spec/integration/database.yml +25 -0
- data/spec/integration/entangler_spec.rb +68 -0
- data/spec/integration/integration_helper.rb +252 -0
- data/spec/integration/invoker_spec.rb +33 -0
- data/spec/integration/lhm_spec.rb +659 -0
- data/spec/integration/lock_wait_timeout_spec.rb +30 -0
- data/spec/integration/locked_switcher_spec.rb +50 -0
- data/spec/integration/proxysql_spec.rb +34 -0
- data/spec/integration/sql_retry/db_connection_helper.rb +52 -0
- data/spec/integration/sql_retry/lock_wait_spec.rb +127 -0
- data/spec/integration/sql_retry/lock_wait_timeout_test_helper.rb +114 -0
- data/spec/integration/sql_retry/proxysql_helper.rb +22 -0
- data/spec/integration/sql_retry/retry_with_proxysql_spec.rb +109 -0
- data/spec/integration/table_spec.rb +83 -0
- data/spec/integration/toxiproxy_helper.rb +40 -0
- data/spec/test_helper.rb +69 -0
- data/spec/unit/atomic_switcher_spec.rb +29 -0
- data/spec/unit/chunk_finder_spec.rb +73 -0
- data/spec/unit/chunk_insert_spec.rb +67 -0
- data/spec/unit/chunker_spec.rb +176 -0
- data/spec/unit/connection_spec.rb +111 -0
- data/spec/unit/entangler_spec.rb +187 -0
- data/spec/unit/intersection_spec.rb +51 -0
- data/spec/unit/lhm_spec.rb +46 -0
- data/spec/unit/locked_switcher_spec.rb +46 -0
- data/spec/unit/migrator_spec.rb +144 -0
- data/spec/unit/printer_spec.rb +85 -0
- data/spec/unit/sql_helper_spec.rb +28 -0
- data/spec/unit/table_name_spec.rb +39 -0
- data/spec/unit/table_spec.rb +47 -0
- data/spec/unit/throttler/slave_lag_spec.rb +322 -0
- data/spec/unit/throttler/threads_running_spec.rb +64 -0
- data/spec/unit/throttler_spec.rb +124 -0
- data/spec/unit/unit_helper.rb +26 -0
- metadata +366 -0
@@ -0,0 +1,252 @@
|
|
1
|
+
# Copyright (c) 2011 - 2013, SoundCloud Ltd., Rany Keddo, Tobias Bielohlawek, Tobias
|
2
|
+
# Schmidt
|
3
|
+
require 'test_helper'
|
4
|
+
require 'yaml'
|
5
|
+
require 'active_support'
|
6
|
+
require 'logger'
|
7
|
+
|
8
|
+
begin
|
9
|
+
$db_config = YAML.load_file(File.expand_path(File.dirname(__FILE__)) + '/database.yml')
|
10
|
+
rescue StandardError => e
|
11
|
+
puts "Run install.sh to setup database"
|
12
|
+
raise e
|
13
|
+
end
|
14
|
+
|
15
|
+
$db_name = 'test'
|
16
|
+
|
17
|
+
require 'lhm/table'
|
18
|
+
require 'lhm/sql_helper'
|
19
|
+
|
20
|
+
module IntegrationHelper
|
21
|
+
|
22
|
+
def self.included(base)
|
23
|
+
base.after(:each) do
|
24
|
+
cleanup_connection = new_mysql_connection
|
25
|
+
results = cleanup_connection.query("SELECT table_name FROM information_schema.tables WHERE table_schema = '#{$db_name}';")
|
26
|
+
table_names_for_cleanup = results.map { |row| "#{$db_name}." + row.values.first }
|
27
|
+
cleanup_connection.query("DROP TABLE IF EXISTS #{table_names_for_cleanup.join(', ')};") if table_names_for_cleanup.length > 0
|
28
|
+
end
|
29
|
+
end
|
30
|
+
|
31
|
+
#
|
32
|
+
# Connectivity
|
33
|
+
#
|
34
|
+
def connection
|
35
|
+
@connection
|
36
|
+
end
|
37
|
+
|
38
|
+
def connect_proxysql!
|
39
|
+
connect!(
|
40
|
+
'127.0.0.1',
|
41
|
+
$db_config['proxysql']['port'],
|
42
|
+
$db_config['proxysql']['user'],
|
43
|
+
$db_config['proxysql']['password'],
|
44
|
+
)
|
45
|
+
end
|
46
|
+
|
47
|
+
def connect_master!
|
48
|
+
connect!(
|
49
|
+
'127.0.0.1',
|
50
|
+
$db_config['master']['port'],
|
51
|
+
$db_config['master']['user'],
|
52
|
+
$db_config['master']['password'],
|
53
|
+
)
|
54
|
+
end
|
55
|
+
|
56
|
+
def connect_slave!
|
57
|
+
connect!(
|
58
|
+
'127.0.0.1',
|
59
|
+
$db_config['slave']['port'],
|
60
|
+
$db_config['slave']['user'],
|
61
|
+
$db_config['slave']['password'],
|
62
|
+
)
|
63
|
+
end
|
64
|
+
|
65
|
+
def connect_master_with_toxiproxy!
|
66
|
+
connect!(
|
67
|
+
'127.0.0.1',
|
68
|
+
$db_config['master_toxic']['port'],
|
69
|
+
$db_config['master_toxic']['user'],
|
70
|
+
$db_config['master_toxic']['password'])
|
71
|
+
end
|
72
|
+
|
73
|
+
def connect!(hostname, port, user, password)
|
74
|
+
Lhm.setup(ar_conn(hostname, port, user, password))
|
75
|
+
unless defined?(@@cleaned_up)
|
76
|
+
Lhm.cleanup(true)
|
77
|
+
@@cleaned_up = true
|
78
|
+
end
|
79
|
+
@connection = Lhm.connection
|
80
|
+
end
|
81
|
+
|
82
|
+
def ar_conn(host, port, user, password)
|
83
|
+
ActiveRecord::Base.establish_connection(
|
84
|
+
:adapter => 'mysql2',
|
85
|
+
:host => host,
|
86
|
+
:username => user,
|
87
|
+
:port => port,
|
88
|
+
:password => password,
|
89
|
+
:database => $db_name
|
90
|
+
)
|
91
|
+
ActiveRecord::Base.connection
|
92
|
+
end
|
93
|
+
|
94
|
+
def select_one(*args)
|
95
|
+
@connection.select_one(*args)
|
96
|
+
end
|
97
|
+
|
98
|
+
def select_value(*args)
|
99
|
+
@connection.select_value(*args)
|
100
|
+
end
|
101
|
+
|
102
|
+
def execute(*args)
|
103
|
+
retries = 10
|
104
|
+
begin
|
105
|
+
@connection.execute(*args)
|
106
|
+
rescue => e
|
107
|
+
if (retries -= 1) > 0 && e.message =~ /Table '.*?' doesn't exist/
|
108
|
+
sleep 0.1
|
109
|
+
retry
|
110
|
+
else
|
111
|
+
raise
|
112
|
+
end
|
113
|
+
end
|
114
|
+
end
|
115
|
+
|
116
|
+
def slave(&block)
|
117
|
+
if master_slave_mode?
|
118
|
+
connect_slave!
|
119
|
+
|
120
|
+
# need to wait for the slave to catch up. a better method would be to
|
121
|
+
# check the master binlog position and wait for the slave to catch up
|
122
|
+
# to that position.
|
123
|
+
sleep 1
|
124
|
+
else
|
125
|
+
connect_master!
|
126
|
+
end
|
127
|
+
|
128
|
+
yield block
|
129
|
+
|
130
|
+
if master_slave_mode?
|
131
|
+
connect_master!
|
132
|
+
end
|
133
|
+
end
|
134
|
+
|
135
|
+
# Helps testing behaviour when another client locks the db
|
136
|
+
def start_locking_thread(lock_for, queue, locking_query)
|
137
|
+
Thread.new do
|
138
|
+
conn = new_mysql_connection
|
139
|
+
conn.query('BEGIN')
|
140
|
+
conn.query(locking_query)
|
141
|
+
queue.push(true)
|
142
|
+
sleep(lock_for) # Sleep for log so LHM gives up
|
143
|
+
conn.query('ROLLBACK')
|
144
|
+
end
|
145
|
+
end
|
146
|
+
|
147
|
+
#
|
148
|
+
# Test Data
|
149
|
+
#
|
150
|
+
|
151
|
+
def fixture(name)
|
152
|
+
File.read($fixtures.join("#{ name }.ddl"))
|
153
|
+
end
|
154
|
+
|
155
|
+
def table_create(fixture_name)
|
156
|
+
execute "drop table if exists `#{ fixture_name }`"
|
157
|
+
execute fixture(fixture_name)
|
158
|
+
table_read(fixture_name)
|
159
|
+
end
|
160
|
+
|
161
|
+
def table_rename(from_name, to_name)
|
162
|
+
execute "rename table `#{ from_name }` to `#{ to_name }`"
|
163
|
+
end
|
164
|
+
|
165
|
+
def table_read(fixture_name)
|
166
|
+
Lhm::Table.parse(fixture_name, @connection)
|
167
|
+
end
|
168
|
+
|
169
|
+
def data_source_exists?(table)
|
170
|
+
connection.data_source_exists?(table.name)
|
171
|
+
end
|
172
|
+
|
173
|
+
def new_mysql_connection(role='master')
|
174
|
+
Mysql2::Client.new(
|
175
|
+
host: '127.0.0.1',
|
176
|
+
database: $db_name,
|
177
|
+
username: $db_config[role]['user'],
|
178
|
+
password: $db_config[role]['password'],
|
179
|
+
port: $db_config[role]['port'],
|
180
|
+
socket: $db_config[role]['socket']
|
181
|
+
)
|
182
|
+
end
|
183
|
+
|
184
|
+
#
|
185
|
+
# Database Helpers
|
186
|
+
#
|
187
|
+
|
188
|
+
def count(table, column, value)
|
189
|
+
query = "select count(*) from #{ table } where #{ column } = '#{ value }'"
|
190
|
+
select_value(query).to_i
|
191
|
+
end
|
192
|
+
|
193
|
+
def count_all(table)
|
194
|
+
query = "select count(*) from `#{ table }`"
|
195
|
+
select_value(query).to_i
|
196
|
+
end
|
197
|
+
|
198
|
+
def index_on_columns?(table_name, cols, type = :non_unique)
|
199
|
+
key_name = Lhm::SqlHelper.idx_name(table_name, cols)
|
200
|
+
|
201
|
+
index?(table_name, key_name, type)
|
202
|
+
end
|
203
|
+
|
204
|
+
def index?(table_name, key_name, type = :non_unique)
|
205
|
+
non_unique = type == :non_unique ? 1 : 0
|
206
|
+
|
207
|
+
!!select_one(%Q<
|
208
|
+
show indexes in `#{ table_name }`
|
209
|
+
where key_name = '#{ key_name }'
|
210
|
+
and non_unique = #{ non_unique }
|
211
|
+
>)
|
212
|
+
end
|
213
|
+
|
214
|
+
#
|
215
|
+
# Environment
|
216
|
+
#
|
217
|
+
|
218
|
+
def master_slave_mode?
|
219
|
+
!!ENV['MASTER_SLAVE']
|
220
|
+
end
|
221
|
+
|
222
|
+
#
|
223
|
+
# Misc
|
224
|
+
#
|
225
|
+
|
226
|
+
def capture_stdout
|
227
|
+
out = StringIO.new
|
228
|
+
$stdout = out
|
229
|
+
logger = Logger.new($stdout)
|
230
|
+
yield logger
|
231
|
+
return out.string
|
232
|
+
ensure
|
233
|
+
$stdout = ::STDOUT
|
234
|
+
end
|
235
|
+
|
236
|
+
def simulate_failed_migration
|
237
|
+
Lhm::Entangler.class_eval do
|
238
|
+
alias_method :old_after, :after
|
239
|
+
def after
|
240
|
+
true
|
241
|
+
end
|
242
|
+
end
|
243
|
+
|
244
|
+
yield
|
245
|
+
ensure
|
246
|
+
Lhm::Entangler.class_eval do
|
247
|
+
undef_method :after
|
248
|
+
alias_method :after, :old_after
|
249
|
+
undef_method :old_after
|
250
|
+
end
|
251
|
+
end
|
252
|
+
end
|
@@ -0,0 +1,33 @@
|
|
1
|
+
require File.expand_path(File.dirname(__FILE__)) + '/integration_helper'
|
2
|
+
|
3
|
+
require 'lhm/invoker'
|
4
|
+
|
5
|
+
describe Lhm::Invoker do
|
6
|
+
include IntegrationHelper
|
7
|
+
|
8
|
+
before(:each) do
|
9
|
+
connect_master!
|
10
|
+
@origin = table_create('users')
|
11
|
+
@destination = table_create('destination')
|
12
|
+
@invoker = Lhm::Invoker.new(Lhm::Table.parse(:users, @connection), @connection)
|
13
|
+
@migration = Lhm::Migration.new(@origin, @destination)
|
14
|
+
@entangler = Lhm::Entangler.new(@migration, @connection)
|
15
|
+
@entangler.before
|
16
|
+
end
|
17
|
+
|
18
|
+
after(:each) do
|
19
|
+
@entangler.after if @invoker.triggers_still_exist?(@invoker.connection, @entangler)
|
20
|
+
end
|
21
|
+
|
22
|
+
describe 'triggers_still_exist?' do
|
23
|
+
it 'should return true when triggers still exist' do
|
24
|
+
assert @invoker.triggers_still_exist?(@invoker.connection, @entangler)
|
25
|
+
end
|
26
|
+
|
27
|
+
it 'should return false when triggers do not exist' do
|
28
|
+
@entangler.after
|
29
|
+
|
30
|
+
refute @invoker.triggers_still_exist?(@invoker.connection, @entangler)
|
31
|
+
end
|
32
|
+
end
|
33
|
+
end
|