lhm-teak 3.6.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (112) hide show
  1. checksums.yaml +7 -0
  2. data/.github/workflows/test.yml +43 -0
  3. data/.gitignore +12 -0
  4. data/.rubocop.yml +183 -0
  5. data/.travis.yml +21 -0
  6. data/Appraisals +24 -0
  7. data/CHANGELOG.md +254 -0
  8. data/Gemfile +5 -0
  9. data/Gemfile.lock +67 -0
  10. data/LICENSE +27 -0
  11. data/README.md +335 -0
  12. data/Rakefile +33 -0
  13. data/dev.yml +45 -0
  14. data/docker-compose.yml +60 -0
  15. data/gemfiles/activerecord_5.2.gemfile +9 -0
  16. data/gemfiles/activerecord_5.2.gemfile.lock +66 -0
  17. data/gemfiles/activerecord_6.0.gemfile +7 -0
  18. data/gemfiles/activerecord_6.0.gemfile.lock +68 -0
  19. data/gemfiles/activerecord_6.1.gemfile +7 -0
  20. data/gemfiles/activerecord_6.1.gemfile.lock +67 -0
  21. data/gemfiles/activerecord_7.0.0.alpha2.gemfile +7 -0
  22. data/gemfiles/activerecord_7.0.0.alpha2.gemfile.lock +65 -0
  23. data/lhm.gemspec +38 -0
  24. data/lib/lhm/atomic_switcher.rb +46 -0
  25. data/lib/lhm/chunk_finder.rb +62 -0
  26. data/lib/lhm/chunk_insert.rb +61 -0
  27. data/lib/lhm/chunker.rb +95 -0
  28. data/lib/lhm/cleanup/current.rb +71 -0
  29. data/lib/lhm/command.rb +48 -0
  30. data/lib/lhm/connection.rb +108 -0
  31. data/lib/lhm/entangler.rb +112 -0
  32. data/lib/lhm/intersection.rb +51 -0
  33. data/lib/lhm/invoker.rb +100 -0
  34. data/lib/lhm/locked_switcher.rb +76 -0
  35. data/lib/lhm/migration.rb +51 -0
  36. data/lib/lhm/migrator.rb +244 -0
  37. data/lib/lhm/printer.rb +63 -0
  38. data/lib/lhm/proxysql_helper.rb +10 -0
  39. data/lib/lhm/railtie.rb +9 -0
  40. data/lib/lhm/sql_helper.rb +77 -0
  41. data/lib/lhm/sql_retry.rb +180 -0
  42. data/lib/lhm/table.rb +121 -0
  43. data/lib/lhm/table_name.rb +23 -0
  44. data/lib/lhm/test_support.rb +35 -0
  45. data/lib/lhm/throttler/slave_lag.rb +162 -0
  46. data/lib/lhm/throttler/threads_running.rb +53 -0
  47. data/lib/lhm/throttler/time.rb +29 -0
  48. data/lib/lhm/throttler.rb +36 -0
  49. data/lib/lhm/timestamp.rb +11 -0
  50. data/lib/lhm/version.rb +6 -0
  51. data/lib/lhm-shopify.rb +1 -0
  52. data/lib/lhm.rb +156 -0
  53. data/scripts/helpers/wait-for-dbs.sh +21 -0
  54. data/scripts/mysql/reader/create_replication.sql +10 -0
  55. data/scripts/mysql/writer/create_test_db.sql +1 -0
  56. data/scripts/mysql/writer/create_users.sql +6 -0
  57. data/scripts/proxysql/proxysql.cnf +117 -0
  58. data/shipit.rubygems.yml +0 -0
  59. data/spec/.lhm.example +4 -0
  60. data/spec/README.md +58 -0
  61. data/spec/fixtures/bigint_table.ddl +4 -0
  62. data/spec/fixtures/composite_primary_key.ddl +6 -0
  63. data/spec/fixtures/composite_primary_key_dest.ddl +6 -0
  64. data/spec/fixtures/custom_primary_key.ddl +6 -0
  65. data/spec/fixtures/custom_primary_key_dest.ddl +6 -0
  66. data/spec/fixtures/destination.ddl +6 -0
  67. data/spec/fixtures/lines.ddl +7 -0
  68. data/spec/fixtures/origin.ddl +6 -0
  69. data/spec/fixtures/permissions.ddl +5 -0
  70. data/spec/fixtures/small_table.ddl +4 -0
  71. data/spec/fixtures/tracks.ddl +5 -0
  72. data/spec/fixtures/users.ddl +14 -0
  73. data/spec/fixtures/wo_id_int_column.ddl +6 -0
  74. data/spec/integration/atomic_switcher_spec.rb +129 -0
  75. data/spec/integration/chunk_insert_spec.rb +30 -0
  76. data/spec/integration/chunker_spec.rb +269 -0
  77. data/spec/integration/cleanup_spec.rb +147 -0
  78. data/spec/integration/database.yml +25 -0
  79. data/spec/integration/entangler_spec.rb +68 -0
  80. data/spec/integration/integration_helper.rb +252 -0
  81. data/spec/integration/invoker_spec.rb +33 -0
  82. data/spec/integration/lhm_spec.rb +659 -0
  83. data/spec/integration/lock_wait_timeout_spec.rb +30 -0
  84. data/spec/integration/locked_switcher_spec.rb +50 -0
  85. data/spec/integration/proxysql_spec.rb +34 -0
  86. data/spec/integration/sql_retry/db_connection_helper.rb +52 -0
  87. data/spec/integration/sql_retry/lock_wait_spec.rb +127 -0
  88. data/spec/integration/sql_retry/lock_wait_timeout_test_helper.rb +114 -0
  89. data/spec/integration/sql_retry/proxysql_helper.rb +22 -0
  90. data/spec/integration/sql_retry/retry_with_proxysql_spec.rb +109 -0
  91. data/spec/integration/table_spec.rb +83 -0
  92. data/spec/integration/toxiproxy_helper.rb +40 -0
  93. data/spec/test_helper.rb +69 -0
  94. data/spec/unit/atomic_switcher_spec.rb +29 -0
  95. data/spec/unit/chunk_finder_spec.rb +73 -0
  96. data/spec/unit/chunk_insert_spec.rb +67 -0
  97. data/spec/unit/chunker_spec.rb +176 -0
  98. data/spec/unit/connection_spec.rb +111 -0
  99. data/spec/unit/entangler_spec.rb +187 -0
  100. data/spec/unit/intersection_spec.rb +51 -0
  101. data/spec/unit/lhm_spec.rb +46 -0
  102. data/spec/unit/locked_switcher_spec.rb +46 -0
  103. data/spec/unit/migrator_spec.rb +144 -0
  104. data/spec/unit/printer_spec.rb +85 -0
  105. data/spec/unit/sql_helper_spec.rb +28 -0
  106. data/spec/unit/table_name_spec.rb +39 -0
  107. data/spec/unit/table_spec.rb +47 -0
  108. data/spec/unit/throttler/slave_lag_spec.rb +322 -0
  109. data/spec/unit/throttler/threads_running_spec.rb +64 -0
  110. data/spec/unit/throttler_spec.rb +124 -0
  111. data/spec/unit/unit_helper.rb +26 -0
  112. metadata +366 -0
@@ -0,0 +1,252 @@
1
+ # Copyright (c) 2011 - 2013, SoundCloud Ltd., Rany Keddo, Tobias Bielohlawek, Tobias
2
+ # Schmidt
3
+ require 'test_helper'
4
+ require 'yaml'
5
+ require 'active_support'
6
+ require 'logger'
7
+
8
+ begin
9
+ $db_config = YAML.load_file(File.expand_path(File.dirname(__FILE__)) + '/database.yml')
10
+ rescue StandardError => e
11
+ puts "Run install.sh to setup database"
12
+ raise e
13
+ end
14
+
15
+ $db_name = 'test'
16
+
17
+ require 'lhm/table'
18
+ require 'lhm/sql_helper'
19
+
20
+ module IntegrationHelper
21
+
22
+ def self.included(base)
23
+ base.after(:each) do
24
+ cleanup_connection = new_mysql_connection
25
+ results = cleanup_connection.query("SELECT table_name FROM information_schema.tables WHERE table_schema = '#{$db_name}';")
26
+ table_names_for_cleanup = results.map { |row| "#{$db_name}." + row.values.first }
27
+ cleanup_connection.query("DROP TABLE IF EXISTS #{table_names_for_cleanup.join(', ')};") if table_names_for_cleanup.length > 0
28
+ end
29
+ end
30
+
31
+ #
32
+ # Connectivity
33
+ #
34
+ def connection
35
+ @connection
36
+ end
37
+
38
+ def connect_proxysql!
39
+ connect!(
40
+ '127.0.0.1',
41
+ $db_config['proxysql']['port'],
42
+ $db_config['proxysql']['user'],
43
+ $db_config['proxysql']['password'],
44
+ )
45
+ end
46
+
47
+ def connect_master!
48
+ connect!(
49
+ '127.0.0.1',
50
+ $db_config['master']['port'],
51
+ $db_config['master']['user'],
52
+ $db_config['master']['password'],
53
+ )
54
+ end
55
+
56
+ def connect_slave!
57
+ connect!(
58
+ '127.0.0.1',
59
+ $db_config['slave']['port'],
60
+ $db_config['slave']['user'],
61
+ $db_config['slave']['password'],
62
+ )
63
+ end
64
+
65
+ def connect_master_with_toxiproxy!
66
+ connect!(
67
+ '127.0.0.1',
68
+ $db_config['master_toxic']['port'],
69
+ $db_config['master_toxic']['user'],
70
+ $db_config['master_toxic']['password'])
71
+ end
72
+
73
+ def connect!(hostname, port, user, password)
74
+ Lhm.setup(ar_conn(hostname, port, user, password))
75
+ unless defined?(@@cleaned_up)
76
+ Lhm.cleanup(true)
77
+ @@cleaned_up = true
78
+ end
79
+ @connection = Lhm.connection
80
+ end
81
+
82
+ def ar_conn(host, port, user, password)
83
+ ActiveRecord::Base.establish_connection(
84
+ :adapter => 'mysql2',
85
+ :host => host,
86
+ :username => user,
87
+ :port => port,
88
+ :password => password,
89
+ :database => $db_name
90
+ )
91
+ ActiveRecord::Base.connection
92
+ end
93
+
94
+ def select_one(*args)
95
+ @connection.select_one(*args)
96
+ end
97
+
98
+ def select_value(*args)
99
+ @connection.select_value(*args)
100
+ end
101
+
102
+ def execute(*args)
103
+ retries = 10
104
+ begin
105
+ @connection.execute(*args)
106
+ rescue => e
107
+ if (retries -= 1) > 0 && e.message =~ /Table '.*?' doesn't exist/
108
+ sleep 0.1
109
+ retry
110
+ else
111
+ raise
112
+ end
113
+ end
114
+ end
115
+
116
+ def slave(&block)
117
+ if master_slave_mode?
118
+ connect_slave!
119
+
120
+ # need to wait for the slave to catch up. a better method would be to
121
+ # check the master binlog position and wait for the slave to catch up
122
+ # to that position.
123
+ sleep 1
124
+ else
125
+ connect_master!
126
+ end
127
+
128
+ yield block
129
+
130
+ if master_slave_mode?
131
+ connect_master!
132
+ end
133
+ end
134
+
135
+ # Helps testing behaviour when another client locks the db
136
+ def start_locking_thread(lock_for, queue, locking_query)
137
+ Thread.new do
138
+ conn = new_mysql_connection
139
+ conn.query('BEGIN')
140
+ conn.query(locking_query)
141
+ queue.push(true)
142
+ sleep(lock_for) # Sleep for log so LHM gives up
143
+ conn.query('ROLLBACK')
144
+ end
145
+ end
146
+
147
+ #
148
+ # Test Data
149
+ #
150
+
151
+ def fixture(name)
152
+ File.read($fixtures.join("#{ name }.ddl"))
153
+ end
154
+
155
+ def table_create(fixture_name)
156
+ execute "drop table if exists `#{ fixture_name }`"
157
+ execute fixture(fixture_name)
158
+ table_read(fixture_name)
159
+ end
160
+
161
+ def table_rename(from_name, to_name)
162
+ execute "rename table `#{ from_name }` to `#{ to_name }`"
163
+ end
164
+
165
+ def table_read(fixture_name)
166
+ Lhm::Table.parse(fixture_name, @connection)
167
+ end
168
+
169
+ def data_source_exists?(table)
170
+ connection.data_source_exists?(table.name)
171
+ end
172
+
173
+ def new_mysql_connection(role='master')
174
+ Mysql2::Client.new(
175
+ host: '127.0.0.1',
176
+ database: $db_name,
177
+ username: $db_config[role]['user'],
178
+ password: $db_config[role]['password'],
179
+ port: $db_config[role]['port'],
180
+ socket: $db_config[role]['socket']
181
+ )
182
+ end
183
+
184
+ #
185
+ # Database Helpers
186
+ #
187
+
188
+ def count(table, column, value)
189
+ query = "select count(*) from #{ table } where #{ column } = '#{ value }'"
190
+ select_value(query).to_i
191
+ end
192
+
193
+ def count_all(table)
194
+ query = "select count(*) from `#{ table }`"
195
+ select_value(query).to_i
196
+ end
197
+
198
+ def index_on_columns?(table_name, cols, type = :non_unique)
199
+ key_name = Lhm::SqlHelper.idx_name(table_name, cols)
200
+
201
+ index?(table_name, key_name, type)
202
+ end
203
+
204
+ def index?(table_name, key_name, type = :non_unique)
205
+ non_unique = type == :non_unique ? 1 : 0
206
+
207
+ !!select_one(%Q<
208
+ show indexes in `#{ table_name }`
209
+ where key_name = '#{ key_name }'
210
+ and non_unique = #{ non_unique }
211
+ >)
212
+ end
213
+
214
+ #
215
+ # Environment
216
+ #
217
+
218
+ def master_slave_mode?
219
+ !!ENV['MASTER_SLAVE']
220
+ end
221
+
222
+ #
223
+ # Misc
224
+ #
225
+
226
+ def capture_stdout
227
+ out = StringIO.new
228
+ $stdout = out
229
+ logger = Logger.new($stdout)
230
+ yield logger
231
+ return out.string
232
+ ensure
233
+ $stdout = ::STDOUT
234
+ end
235
+
236
+ def simulate_failed_migration
237
+ Lhm::Entangler.class_eval do
238
+ alias_method :old_after, :after
239
+ def after
240
+ true
241
+ end
242
+ end
243
+
244
+ yield
245
+ ensure
246
+ Lhm::Entangler.class_eval do
247
+ undef_method :after
248
+ alias_method :after, :old_after
249
+ undef_method :old_after
250
+ end
251
+ end
252
+ end
@@ -0,0 +1,33 @@
1
+ require File.expand_path(File.dirname(__FILE__)) + '/integration_helper'
2
+
3
+ require 'lhm/invoker'
4
+
5
+ describe Lhm::Invoker do
6
+ include IntegrationHelper
7
+
8
+ before(:each) do
9
+ connect_master!
10
+ @origin = table_create('users')
11
+ @destination = table_create('destination')
12
+ @invoker = Lhm::Invoker.new(Lhm::Table.parse(:users, @connection), @connection)
13
+ @migration = Lhm::Migration.new(@origin, @destination)
14
+ @entangler = Lhm::Entangler.new(@migration, @connection)
15
+ @entangler.before
16
+ end
17
+
18
+ after(:each) do
19
+ @entangler.after if @invoker.triggers_still_exist?(@invoker.connection, @entangler)
20
+ end
21
+
22
+ describe 'triggers_still_exist?' do
23
+ it 'should return true when triggers still exist' do
24
+ assert @invoker.triggers_still_exist?(@invoker.connection, @entangler)
25
+ end
26
+
27
+ it 'should return false when triggers do not exist' do
28
+ @entangler.after
29
+
30
+ refute @invoker.triggers_still_exist?(@invoker.connection, @entangler)
31
+ end
32
+ end
33
+ end