lhm-teak 3.6.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (112) hide show
  1. checksums.yaml +7 -0
  2. data/.github/workflows/test.yml +43 -0
  3. data/.gitignore +12 -0
  4. data/.rubocop.yml +183 -0
  5. data/.travis.yml +21 -0
  6. data/Appraisals +24 -0
  7. data/CHANGELOG.md +254 -0
  8. data/Gemfile +5 -0
  9. data/Gemfile.lock +67 -0
  10. data/LICENSE +27 -0
  11. data/README.md +335 -0
  12. data/Rakefile +33 -0
  13. data/dev.yml +45 -0
  14. data/docker-compose.yml +60 -0
  15. data/gemfiles/activerecord_5.2.gemfile +9 -0
  16. data/gemfiles/activerecord_5.2.gemfile.lock +66 -0
  17. data/gemfiles/activerecord_6.0.gemfile +7 -0
  18. data/gemfiles/activerecord_6.0.gemfile.lock +68 -0
  19. data/gemfiles/activerecord_6.1.gemfile +7 -0
  20. data/gemfiles/activerecord_6.1.gemfile.lock +67 -0
  21. data/gemfiles/activerecord_7.0.0.alpha2.gemfile +7 -0
  22. data/gemfiles/activerecord_7.0.0.alpha2.gemfile.lock +65 -0
  23. data/lhm.gemspec +38 -0
  24. data/lib/lhm/atomic_switcher.rb +46 -0
  25. data/lib/lhm/chunk_finder.rb +62 -0
  26. data/lib/lhm/chunk_insert.rb +61 -0
  27. data/lib/lhm/chunker.rb +95 -0
  28. data/lib/lhm/cleanup/current.rb +71 -0
  29. data/lib/lhm/command.rb +48 -0
  30. data/lib/lhm/connection.rb +108 -0
  31. data/lib/lhm/entangler.rb +112 -0
  32. data/lib/lhm/intersection.rb +51 -0
  33. data/lib/lhm/invoker.rb +100 -0
  34. data/lib/lhm/locked_switcher.rb +76 -0
  35. data/lib/lhm/migration.rb +51 -0
  36. data/lib/lhm/migrator.rb +244 -0
  37. data/lib/lhm/printer.rb +63 -0
  38. data/lib/lhm/proxysql_helper.rb +10 -0
  39. data/lib/lhm/railtie.rb +9 -0
  40. data/lib/lhm/sql_helper.rb +77 -0
  41. data/lib/lhm/sql_retry.rb +180 -0
  42. data/lib/lhm/table.rb +121 -0
  43. data/lib/lhm/table_name.rb +23 -0
  44. data/lib/lhm/test_support.rb +35 -0
  45. data/lib/lhm/throttler/slave_lag.rb +162 -0
  46. data/lib/lhm/throttler/threads_running.rb +53 -0
  47. data/lib/lhm/throttler/time.rb +29 -0
  48. data/lib/lhm/throttler.rb +36 -0
  49. data/lib/lhm/timestamp.rb +11 -0
  50. data/lib/lhm/version.rb +6 -0
  51. data/lib/lhm-shopify.rb +1 -0
  52. data/lib/lhm.rb +156 -0
  53. data/scripts/helpers/wait-for-dbs.sh +21 -0
  54. data/scripts/mysql/reader/create_replication.sql +10 -0
  55. data/scripts/mysql/writer/create_test_db.sql +1 -0
  56. data/scripts/mysql/writer/create_users.sql +6 -0
  57. data/scripts/proxysql/proxysql.cnf +117 -0
  58. data/shipit.rubygems.yml +0 -0
  59. data/spec/.lhm.example +4 -0
  60. data/spec/README.md +58 -0
  61. data/spec/fixtures/bigint_table.ddl +4 -0
  62. data/spec/fixtures/composite_primary_key.ddl +6 -0
  63. data/spec/fixtures/composite_primary_key_dest.ddl +6 -0
  64. data/spec/fixtures/custom_primary_key.ddl +6 -0
  65. data/spec/fixtures/custom_primary_key_dest.ddl +6 -0
  66. data/spec/fixtures/destination.ddl +6 -0
  67. data/spec/fixtures/lines.ddl +7 -0
  68. data/spec/fixtures/origin.ddl +6 -0
  69. data/spec/fixtures/permissions.ddl +5 -0
  70. data/spec/fixtures/small_table.ddl +4 -0
  71. data/spec/fixtures/tracks.ddl +5 -0
  72. data/spec/fixtures/users.ddl +14 -0
  73. data/spec/fixtures/wo_id_int_column.ddl +6 -0
  74. data/spec/integration/atomic_switcher_spec.rb +129 -0
  75. data/spec/integration/chunk_insert_spec.rb +30 -0
  76. data/spec/integration/chunker_spec.rb +269 -0
  77. data/spec/integration/cleanup_spec.rb +147 -0
  78. data/spec/integration/database.yml +25 -0
  79. data/spec/integration/entangler_spec.rb +68 -0
  80. data/spec/integration/integration_helper.rb +252 -0
  81. data/spec/integration/invoker_spec.rb +33 -0
  82. data/spec/integration/lhm_spec.rb +659 -0
  83. data/spec/integration/lock_wait_timeout_spec.rb +30 -0
  84. data/spec/integration/locked_switcher_spec.rb +50 -0
  85. data/spec/integration/proxysql_spec.rb +34 -0
  86. data/spec/integration/sql_retry/db_connection_helper.rb +52 -0
  87. data/spec/integration/sql_retry/lock_wait_spec.rb +127 -0
  88. data/spec/integration/sql_retry/lock_wait_timeout_test_helper.rb +114 -0
  89. data/spec/integration/sql_retry/proxysql_helper.rb +22 -0
  90. data/spec/integration/sql_retry/retry_with_proxysql_spec.rb +109 -0
  91. data/spec/integration/table_spec.rb +83 -0
  92. data/spec/integration/toxiproxy_helper.rb +40 -0
  93. data/spec/test_helper.rb +69 -0
  94. data/spec/unit/atomic_switcher_spec.rb +29 -0
  95. data/spec/unit/chunk_finder_spec.rb +73 -0
  96. data/spec/unit/chunk_insert_spec.rb +67 -0
  97. data/spec/unit/chunker_spec.rb +176 -0
  98. data/spec/unit/connection_spec.rb +111 -0
  99. data/spec/unit/entangler_spec.rb +187 -0
  100. data/spec/unit/intersection_spec.rb +51 -0
  101. data/spec/unit/lhm_spec.rb +46 -0
  102. data/spec/unit/locked_switcher_spec.rb +46 -0
  103. data/spec/unit/migrator_spec.rb +144 -0
  104. data/spec/unit/printer_spec.rb +85 -0
  105. data/spec/unit/sql_helper_spec.rb +28 -0
  106. data/spec/unit/table_name_spec.rb +39 -0
  107. data/spec/unit/table_spec.rb +47 -0
  108. data/spec/unit/throttler/slave_lag_spec.rb +322 -0
  109. data/spec/unit/throttler/threads_running_spec.rb +64 -0
  110. data/spec/unit/throttler_spec.rb +124 -0
  111. data/spec/unit/unit_helper.rb +26 -0
  112. metadata +366 -0
@@ -0,0 +1,252 @@
1
+ # Copyright (c) 2011 - 2013, SoundCloud Ltd., Rany Keddo, Tobias Bielohlawek, Tobias
2
+ # Schmidt
3
+ require 'test_helper'
4
+ require 'yaml'
5
+ require 'active_support'
6
+ require 'logger'
7
+
8
+ begin
9
+ $db_config = YAML.load_file(File.expand_path(File.dirname(__FILE__)) + '/database.yml')
10
+ rescue StandardError => e
11
+ puts "Run install.sh to setup database"
12
+ raise e
13
+ end
14
+
15
+ $db_name = 'test'
16
+
17
+ require 'lhm/table'
18
+ require 'lhm/sql_helper'
19
+
20
+ module IntegrationHelper
21
+
22
+ def self.included(base)
23
+ base.after(:each) do
24
+ cleanup_connection = new_mysql_connection
25
+ results = cleanup_connection.query("SELECT table_name FROM information_schema.tables WHERE table_schema = '#{$db_name}';")
26
+ table_names_for_cleanup = results.map { |row| "#{$db_name}." + row.values.first }
27
+ cleanup_connection.query("DROP TABLE IF EXISTS #{table_names_for_cleanup.join(', ')};") if table_names_for_cleanup.length > 0
28
+ end
29
+ end
30
+
31
+ #
32
+ # Connectivity
33
+ #
34
+ def connection
35
+ @connection
36
+ end
37
+
38
+ def connect_proxysql!
39
+ connect!(
40
+ '127.0.0.1',
41
+ $db_config['proxysql']['port'],
42
+ $db_config['proxysql']['user'],
43
+ $db_config['proxysql']['password'],
44
+ )
45
+ end
46
+
47
+ def connect_master!
48
+ connect!(
49
+ '127.0.0.1',
50
+ $db_config['master']['port'],
51
+ $db_config['master']['user'],
52
+ $db_config['master']['password'],
53
+ )
54
+ end
55
+
56
+ def connect_slave!
57
+ connect!(
58
+ '127.0.0.1',
59
+ $db_config['slave']['port'],
60
+ $db_config['slave']['user'],
61
+ $db_config['slave']['password'],
62
+ )
63
+ end
64
+
65
+ def connect_master_with_toxiproxy!
66
+ connect!(
67
+ '127.0.0.1',
68
+ $db_config['master_toxic']['port'],
69
+ $db_config['master_toxic']['user'],
70
+ $db_config['master_toxic']['password'])
71
+ end
72
+
73
+ def connect!(hostname, port, user, password)
74
+ Lhm.setup(ar_conn(hostname, port, user, password))
75
+ unless defined?(@@cleaned_up)
76
+ Lhm.cleanup(true)
77
+ @@cleaned_up = true
78
+ end
79
+ @connection = Lhm.connection
80
+ end
81
+
82
+ def ar_conn(host, port, user, password)
83
+ ActiveRecord::Base.establish_connection(
84
+ :adapter => 'mysql2',
85
+ :host => host,
86
+ :username => user,
87
+ :port => port,
88
+ :password => password,
89
+ :database => $db_name
90
+ )
91
+ ActiveRecord::Base.connection
92
+ end
93
+
94
+ def select_one(*args)
95
+ @connection.select_one(*args)
96
+ end
97
+
98
+ def select_value(*args)
99
+ @connection.select_value(*args)
100
+ end
101
+
102
+ def execute(*args)
103
+ retries = 10
104
+ begin
105
+ @connection.execute(*args)
106
+ rescue => e
107
+ if (retries -= 1) > 0 && e.message =~ /Table '.*?' doesn't exist/
108
+ sleep 0.1
109
+ retry
110
+ else
111
+ raise
112
+ end
113
+ end
114
+ end
115
+
116
+ def slave(&block)
117
+ if master_slave_mode?
118
+ connect_slave!
119
+
120
+ # need to wait for the slave to catch up. a better method would be to
121
+ # check the master binlog position and wait for the slave to catch up
122
+ # to that position.
123
+ sleep 1
124
+ else
125
+ connect_master!
126
+ end
127
+
128
+ yield block
129
+
130
+ if master_slave_mode?
131
+ connect_master!
132
+ end
133
+ end
134
+
135
+ # Helps testing behaviour when another client locks the db
136
+ def start_locking_thread(lock_for, queue, locking_query)
137
+ Thread.new do
138
+ conn = new_mysql_connection
139
+ conn.query('BEGIN')
140
+ conn.query(locking_query)
141
+ queue.push(true)
142
+ sleep(lock_for) # Sleep for log so LHM gives up
143
+ conn.query('ROLLBACK')
144
+ end
145
+ end
146
+
147
+ #
148
+ # Test Data
149
+ #
150
+
151
+ def fixture(name)
152
+ File.read($fixtures.join("#{ name }.ddl"))
153
+ end
154
+
155
+ def table_create(fixture_name)
156
+ execute "drop table if exists `#{ fixture_name }`"
157
+ execute fixture(fixture_name)
158
+ table_read(fixture_name)
159
+ end
160
+
161
+ def table_rename(from_name, to_name)
162
+ execute "rename table `#{ from_name }` to `#{ to_name }`"
163
+ end
164
+
165
+ def table_read(fixture_name)
166
+ Lhm::Table.parse(fixture_name, @connection)
167
+ end
168
+
169
+ def data_source_exists?(table)
170
+ connection.data_source_exists?(table.name)
171
+ end
172
+
173
+ def new_mysql_connection(role='master')
174
+ Mysql2::Client.new(
175
+ host: '127.0.0.1',
176
+ database: $db_name,
177
+ username: $db_config[role]['user'],
178
+ password: $db_config[role]['password'],
179
+ port: $db_config[role]['port'],
180
+ socket: $db_config[role]['socket']
181
+ )
182
+ end
183
+
184
+ #
185
+ # Database Helpers
186
+ #
187
+
188
+ def count(table, column, value)
189
+ query = "select count(*) from #{ table } where #{ column } = '#{ value }'"
190
+ select_value(query).to_i
191
+ end
192
+
193
+ def count_all(table)
194
+ query = "select count(*) from `#{ table }`"
195
+ select_value(query).to_i
196
+ end
197
+
198
+ def index_on_columns?(table_name, cols, type = :non_unique)
199
+ key_name = Lhm::SqlHelper.idx_name(table_name, cols)
200
+
201
+ index?(table_name, key_name, type)
202
+ end
203
+
204
+ def index?(table_name, key_name, type = :non_unique)
205
+ non_unique = type == :non_unique ? 1 : 0
206
+
207
+ !!select_one(%Q<
208
+ show indexes in `#{ table_name }`
209
+ where key_name = '#{ key_name }'
210
+ and non_unique = #{ non_unique }
211
+ >)
212
+ end
213
+
214
+ #
215
+ # Environment
216
+ #
217
+
218
+ def master_slave_mode?
219
+ !!ENV['MASTER_SLAVE']
220
+ end
221
+
222
+ #
223
+ # Misc
224
+ #
225
+
226
+ def capture_stdout
227
+ out = StringIO.new
228
+ $stdout = out
229
+ logger = Logger.new($stdout)
230
+ yield logger
231
+ return out.string
232
+ ensure
233
+ $stdout = ::STDOUT
234
+ end
235
+
236
+ def simulate_failed_migration
237
+ Lhm::Entangler.class_eval do
238
+ alias_method :old_after, :after
239
+ def after
240
+ true
241
+ end
242
+ end
243
+
244
+ yield
245
+ ensure
246
+ Lhm::Entangler.class_eval do
247
+ undef_method :after
248
+ alias_method :after, :old_after
249
+ undef_method :old_after
250
+ end
251
+ end
252
+ end
@@ -0,0 +1,33 @@
1
+ require File.expand_path(File.dirname(__FILE__)) + '/integration_helper'
2
+
3
+ require 'lhm/invoker'
4
+
5
+ describe Lhm::Invoker do
6
+ include IntegrationHelper
7
+
8
+ before(:each) do
9
+ connect_master!
10
+ @origin = table_create('users')
11
+ @destination = table_create('destination')
12
+ @invoker = Lhm::Invoker.new(Lhm::Table.parse(:users, @connection), @connection)
13
+ @migration = Lhm::Migration.new(@origin, @destination)
14
+ @entangler = Lhm::Entangler.new(@migration, @connection)
15
+ @entangler.before
16
+ end
17
+
18
+ after(:each) do
19
+ @entangler.after if @invoker.triggers_still_exist?(@invoker.connection, @entangler)
20
+ end
21
+
22
+ describe 'triggers_still_exist?' do
23
+ it 'should return true when triggers still exist' do
24
+ assert @invoker.triggers_still_exist?(@invoker.connection, @entangler)
25
+ end
26
+
27
+ it 'should return false when triggers do not exist' do
28
+ @entangler.after
29
+
30
+ refute @invoker.triggers_still_exist?(@invoker.connection, @entangler)
31
+ end
32
+ end
33
+ end