migration-patterns 0.0.0.pre.rc1
Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
---
|
2
|
+
SHA256:
|
3
|
+
metadata.gz: 865fc1328c592bc00258c83832c12f7ed83757f0202868d8532d7dd30585d57d
|
4
|
+
data.tar.gz: 88c9961f45cbdce525167682ade3e60a99b25f70dc40a9016313580bbf420237
|
5
|
+
SHA512:
|
6
|
+
metadata.gz: 9b6f7ad7f9224be6ead4110071034363e0b9434cc6dc202f9f6b96e2c3962fb15ba0b78f5395ed3a46ef545ad94c895f5f897281d3c77a2b84e6baa121be8242
|
7
|
+
data.tar.gz: e88d6293f73a04a59c9c46a1ce4fd9fff604d33e9d0d93a8e71310e95d2e043f7b319921821123b4d2674849815d8b430fdfd18e459cbaf897641987a7d9cf25
|
@@ -0,0 +1,41 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module MigrationPatterns
|
4
|
+
module DatabaseHelpers
|
5
|
+
def self.adapter_name
|
6
|
+
config['adapter']
|
7
|
+
end
|
8
|
+
|
9
|
+
def self.postgresql?
|
10
|
+
adapter_name.casecmp('postgresql').zero?
|
11
|
+
end
|
12
|
+
|
13
|
+
def self.mysql?
|
14
|
+
adapter_name.casecmp('mysql2').zero?
|
15
|
+
end
|
16
|
+
|
17
|
+
def self.true_value
|
18
|
+
if postgresql?
|
19
|
+
"'t'"
|
20
|
+
else
|
21
|
+
1
|
22
|
+
end
|
23
|
+
end
|
24
|
+
|
25
|
+
def self.false_value
|
26
|
+
if postgresql?
|
27
|
+
"'f'"
|
28
|
+
else
|
29
|
+
0
|
30
|
+
end
|
31
|
+
end
|
32
|
+
|
33
|
+
def self.username
|
34
|
+
config['username'] || ENV['USER']
|
35
|
+
end
|
36
|
+
|
37
|
+
def self.database_name
|
38
|
+
config['database']
|
39
|
+
end
|
40
|
+
end
|
41
|
+
end
|
@@ -0,0 +1,1078 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module MigrationPatterns
|
4
|
+
module MigrationHelpers
|
5
|
+
BACKGROUND_MIGRATION_BATCH_SIZE = 1000 # Number of rows to process per job
|
6
|
+
BACKGROUND_MIGRATION_JOB_BUFFER_SIZE = 1000 # Number of jobs to bulk queue at a time
|
7
|
+
|
8
|
+
# Adds `created_at` and `updated_at` columns with timezone information.
|
9
|
+
#
|
10
|
+
# This method is an improved version of Rails' built-in method `add_timestamps`.
|
11
|
+
#
|
12
|
+
# Available options are:
|
13
|
+
# default - The default value for the column.
|
14
|
+
# null - When set to `true` the column will allow NULL values.
|
15
|
+
# The default is to not allow NULL values.
|
16
|
+
def add_timestamps_with_timezone(table_name, options = {})
|
17
|
+
options[:null] = false if options[:null].nil?
|
18
|
+
|
19
|
+
%i[created_at updated_at].each do |column_name|
|
20
|
+
if options[:default] && transaction_open?
|
21
|
+
raise '`add_timestamps_with_timezone` with default value cannot be run inside a transaction. ' \
|
22
|
+
'You can disable transactions by calling `disable_ddl_transaction!` ' \
|
23
|
+
'in the body of your migration class'
|
24
|
+
end
|
25
|
+
|
26
|
+
# If default value is presented, use `add_column_with_default` method instead.
|
27
|
+
if options[:default]
|
28
|
+
add_column_with_default(
|
29
|
+
table_name,
|
30
|
+
column_name,
|
31
|
+
:datetime_with_timezone,
|
32
|
+
default: options[:default],
|
33
|
+
allow_null: options[:null]
|
34
|
+
)
|
35
|
+
else
|
36
|
+
add_column(table_name, column_name, :datetime_with_timezone, options)
|
37
|
+
end
|
38
|
+
end
|
39
|
+
end
|
40
|
+
|
41
|
+
# Creates a new index, concurrently when supported
|
42
|
+
#
|
43
|
+
# On PostgreSQL this method creates an index concurrently, on MySQL this
|
44
|
+
# creates a regular index.
|
45
|
+
#
|
46
|
+
# Example:
|
47
|
+
#
|
48
|
+
# add_concurrent_index :users, :some_column
|
49
|
+
#
|
50
|
+
# See Rails' `add_index` for more info on the available arguments.
|
51
|
+
def add_concurrent_index(table_name, column_name, options = {})
|
52
|
+
if transaction_open?
|
53
|
+
raise 'add_concurrent_index can not be run inside a transaction, ' \
|
54
|
+
'you can disable transactions by calling disable_ddl_transaction! ' \
|
55
|
+
'in the body of your migration class'
|
56
|
+
end
|
57
|
+
|
58
|
+
options = options.merge(algorithm: :concurrently) if Database.postgresql?
|
59
|
+
|
60
|
+
if index_exists?(table_name, column_name, options)
|
61
|
+
Rails.logger.warn "Index not created because it already exists (this may be due to an aborted migration or similar): table_name: #{table_name}, column_name: #{column_name}"
|
62
|
+
return
|
63
|
+
end
|
64
|
+
|
65
|
+
disable_statement_timeout do
|
66
|
+
add_index(table_name, column_name, options)
|
67
|
+
end
|
68
|
+
end
|
69
|
+
|
70
|
+
# Removes an existed index, concurrently when supported
|
71
|
+
#
|
72
|
+
# On PostgreSQL this method removes an index concurrently.
|
73
|
+
#
|
74
|
+
# Example:
|
75
|
+
#
|
76
|
+
# remove_concurrent_index :users, :some_column
|
77
|
+
#
|
78
|
+
# See Rails' `remove_index` for more info on the available arguments.
|
79
|
+
def remove_concurrent_index(table_name, column_name, options = {})
|
80
|
+
if transaction_open?
|
81
|
+
raise 'remove_concurrent_index can not be run inside a transaction, ' \
|
82
|
+
'you can disable transactions by calling disable_ddl_transaction! ' \
|
83
|
+
'in the body of your migration class'
|
84
|
+
end
|
85
|
+
|
86
|
+
options = options.merge(algorithm: :concurrently) if supports_drop_index_concurrently?
|
87
|
+
|
88
|
+
unless index_exists?(table_name, column_name, options)
|
89
|
+
Rails.logger.warn "Index not removed because it does not exist (this may be due to an aborted migration or similar): table_name: #{table_name}, column_name: #{column_name}"
|
90
|
+
return
|
91
|
+
end
|
92
|
+
|
93
|
+
disable_statement_timeout do
|
94
|
+
remove_index(table_name, options.merge(column: column_name))
|
95
|
+
end
|
96
|
+
end
|
97
|
+
|
98
|
+
# Removes an existing index, concurrently when supported
|
99
|
+
#
|
100
|
+
# On PostgreSQL this method removes an index concurrently.
|
101
|
+
#
|
102
|
+
# Example:
|
103
|
+
#
|
104
|
+
# remove_concurrent_index :users, "index_X_by_Y"
|
105
|
+
#
|
106
|
+
# See Rails' `remove_index` for more info on the available arguments.
|
107
|
+
def remove_concurrent_index_by_name(table_name, index_name, options = {})
|
108
|
+
if transaction_open?
|
109
|
+
raise 'remove_concurrent_index_by_name can not be run inside a transaction, ' \
|
110
|
+
'you can disable transactions by calling disable_ddl_transaction! ' \
|
111
|
+
'in the body of your migration class'
|
112
|
+
end
|
113
|
+
|
114
|
+
options = options.merge(algorithm: :concurrently) if supports_drop_index_concurrently?
|
115
|
+
|
116
|
+
unless index_exists_by_name?(table_name, index_name)
|
117
|
+
Rails.logger.warn "Index not removed because it does not exist (this may be due to an aborted migration or similar): table_name: #{table_name}, index_name: #{index_name}"
|
118
|
+
return
|
119
|
+
end
|
120
|
+
|
121
|
+
disable_statement_timeout do
|
122
|
+
remove_index(table_name, options.merge(name: index_name))
|
123
|
+
end
|
124
|
+
end
|
125
|
+
|
126
|
+
# Only available on Postgresql >= 9.2
|
127
|
+
def supports_drop_index_concurrently?
|
128
|
+
return false unless Database.postgresql?
|
129
|
+
|
130
|
+
version = select_one("SELECT current_setting('server_version_num') AS v")['v'].to_i
|
131
|
+
|
132
|
+
version >= 90_200
|
133
|
+
end
|
134
|
+
|
135
|
+
# Adds a foreign key with only minimal locking on the tables involved.
|
136
|
+
#
|
137
|
+
# This method only requires minimal locking when using PostgreSQL. When
|
138
|
+
# using MySQL this method will use Rails' default `add_foreign_key`.
|
139
|
+
#
|
140
|
+
# source - The source table containing the foreign key.
|
141
|
+
# target - The target table the key points to.
|
142
|
+
# column - The name of the column to create the foreign key on.
|
143
|
+
# on_delete - The action to perform when associated data is removed,
|
144
|
+
# defaults to "CASCADE".
|
145
|
+
def add_concurrent_foreign_key(source, target, column:, on_delete: :cascade)
|
146
|
+
# Transactions would result in ALTER TABLE locks being held for the
|
147
|
+
# duration of the transaction, defeating the purpose of this method.
|
148
|
+
raise 'add_concurrent_foreign_key can not be run inside a transaction' if transaction_open?
|
149
|
+
|
150
|
+
# While MySQL does allow disabling of foreign keys it has no equivalent
|
151
|
+
# of PostgreSQL's "VALIDATE CONSTRAINT". As a result we'll just fall
|
152
|
+
# back to the normal foreign key procedure.
|
153
|
+
if Database.mysql?
|
154
|
+
if foreign_key_exists?(source, target, column: column)
|
155
|
+
Rails.logger.warn 'Foreign key not created because it exists already ' \
|
156
|
+
'(this may be due to an aborted migration or similar): ' \
|
157
|
+
"source: #{source}, target: #{target}, column: #{column}"
|
158
|
+
return
|
159
|
+
end
|
160
|
+
|
161
|
+
return add_foreign_key(source, target,
|
162
|
+
column: column,
|
163
|
+
on_delete: on_delete)
|
164
|
+
else
|
165
|
+
on_delete = 'SET NULL' if on_delete == :nullify
|
166
|
+
end
|
167
|
+
|
168
|
+
key_name = concurrent_foreign_key_name(source, column)
|
169
|
+
|
170
|
+
unless foreign_key_exists?(source, target, column: column)
|
171
|
+
Rails.logger.warn 'Foreign key not created because it exists already ' \
|
172
|
+
'(this may be due to an aborted migration or similar): ' \
|
173
|
+
"source: #{source}, target: #{target}, column: #{column}"
|
174
|
+
|
175
|
+
# Using NOT VALID allows us to create a key without immediately
|
176
|
+
# validating it. This means we keep the ALTER TABLE lock only for a
|
177
|
+
# short period of time. The key _is_ enforced for any newly created
|
178
|
+
# data.
|
179
|
+
execute <<-EOF.strip_heredoc
|
180
|
+
ALTER TABLE #{source}
|
181
|
+
ADD CONSTRAINT #{key_name}
|
182
|
+
FOREIGN KEY (#{column})
|
183
|
+
REFERENCES #{target} (id)
|
184
|
+
#{on_delete ? "ON DELETE #{on_delete.upcase}" : ''}
|
185
|
+
NOT VALID;
|
186
|
+
EOF
|
187
|
+
end
|
188
|
+
|
189
|
+
# Validate the existing constraint. This can potentially take a very
|
190
|
+
# long time to complete, but fortunately does not lock the source table
|
191
|
+
# while running.
|
192
|
+
#
|
193
|
+
# Note this is a no-op in case the constraint is VALID already
|
194
|
+
disable_statement_timeout do
|
195
|
+
execute("ALTER TABLE #{source} VALIDATE CONSTRAINT #{key_name};")
|
196
|
+
end
|
197
|
+
end
|
198
|
+
|
199
|
+
def foreign_key_exists?(source, target = nil, column: nil)
|
200
|
+
foreign_keys(source).any? do |key|
|
201
|
+
if column
|
202
|
+
key.options[:column].to_s == column.to_s
|
203
|
+
else
|
204
|
+
key.to_table.to_s == target.to_s
|
205
|
+
end
|
206
|
+
end
|
207
|
+
end
|
208
|
+
|
209
|
+
# Returns the name for a concurrent foreign key.
|
210
|
+
#
|
211
|
+
# PostgreSQL constraint names have a limit of 63 bytes. The logic used
|
212
|
+
# here is based on Rails' foreign_key_name() method, which unfortunately
|
213
|
+
# is private so we can't rely on it directly.
|
214
|
+
def concurrent_foreign_key_name(table, column)
|
215
|
+
"fk_#{Digest::SHA256.hexdigest("#{table}_#{column}_fk").first(10)}"
|
216
|
+
end
|
217
|
+
|
218
|
+
# Long-running migrations may take more than the timeout allowed by
|
219
|
+
# the database. Disable the session's statement timeout to ensure
|
220
|
+
# migrations don't get killed prematurely. (PostgreSQL only)
|
221
|
+
#
|
222
|
+
# There are two possible ways to disable the statement timeout:
|
223
|
+
#
|
224
|
+
# - Per transaction (this is the preferred and default mode)
|
225
|
+
# - Per connection (requires a cleanup after the execution)
|
226
|
+
#
|
227
|
+
# When using a per connection disable statement, code must be inside
|
228
|
+
# a block so we can automatically execute `RESET ALL` after block finishes
|
229
|
+
# otherwise the statement will still be disabled until connection is dropped
|
230
|
+
# or `RESET ALL` is executed
|
231
|
+
def disable_statement_timeout
|
232
|
+
# bypass disabled_statement logic when not using postgres, but still execute block when one is given
|
233
|
+
unless Database.postgresql?
|
234
|
+
yield if block_given?
|
235
|
+
|
236
|
+
return
|
237
|
+
end
|
238
|
+
|
239
|
+
if block_given?
|
240
|
+
begin
|
241
|
+
execute('SET statement_timeout TO 0')
|
242
|
+
|
243
|
+
yield
|
244
|
+
ensure
|
245
|
+
execute('RESET ALL')
|
246
|
+
end
|
247
|
+
else
|
248
|
+
unless transaction_open?
|
249
|
+
raise <<~ERROR
|
250
|
+
Cannot call disable_statement_timeout() without a transaction open or outside of a transaction block.
|
251
|
+
If you don't want to use a transaction wrap your code in a block call:
|
252
|
+
|
253
|
+
disable_statement_timeout { # code that requires disabled statement here }
|
254
|
+
|
255
|
+
This will make sure statement_timeout is disabled before and reset after the block execution is finished.
|
256
|
+
ERROR
|
257
|
+
end
|
258
|
+
|
259
|
+
execute('SET LOCAL statement_timeout TO 0')
|
260
|
+
end
|
261
|
+
end
|
262
|
+
|
263
|
+
def true_value
|
264
|
+
Database.true_value
|
265
|
+
end
|
266
|
+
|
267
|
+
def false_value
|
268
|
+
Database.false_value
|
269
|
+
end
|
270
|
+
|
271
|
+
# Updates the value of a column in batches.
|
272
|
+
#
|
273
|
+
# This method updates the table in batches of 5% of the total row count.
|
274
|
+
# A `batch_size` option can also be passed to set this to a fixed number.
|
275
|
+
# This method will continue updating rows until no rows remain.
|
276
|
+
#
|
277
|
+
# When given a block this method will yield two values to the block:
|
278
|
+
#
|
279
|
+
# 1. An instance of `Arel::Table` for the table that is being updated.
|
280
|
+
# 2. The query to run as an Arel object.
|
281
|
+
#
|
282
|
+
# By supplying a block one can add extra conditions to the queries being
|
283
|
+
# executed. Note that the same block is used for _all_ queries.
|
284
|
+
#
|
285
|
+
# Example:
|
286
|
+
#
|
287
|
+
# update_column_in_batches(:projects, :foo, 10) do |table, query|
|
288
|
+
# query.where(table[:some_column].eq('hello'))
|
289
|
+
# end
|
290
|
+
#
|
291
|
+
# This would result in this method updating only rows where
|
292
|
+
# `projects.some_column` equals "hello".
|
293
|
+
#
|
294
|
+
# table - The name of the table.
|
295
|
+
# column - The name of the column to update.
|
296
|
+
# value - The value for the column.
|
297
|
+
#
|
298
|
+
# The `value` argument is typically a literal. To perform a computed
|
299
|
+
# update, an Arel literal can be used instead:
|
300
|
+
#
|
301
|
+
# update_value = Arel.sql('bar * baz')
|
302
|
+
#
|
303
|
+
# update_column_in_batches(:projects, :foo, update_value) do |table, query|
|
304
|
+
# query.where(table[:some_column].eq('hello'))
|
305
|
+
# end
|
306
|
+
#
|
307
|
+
# Rubocop's Metrics/AbcSize metric is disabled for this method as Rubocop
|
308
|
+
# determines this method to be too complex while there's no way to make it
|
309
|
+
# less "complex" without introducing extra methods (which actually will
|
310
|
+
# make things _more_ complex).
|
311
|
+
#
|
312
|
+
# rubocop: disable Metrics/AbcSize
|
313
|
+
def update_column_in_batches(table, column, value, batch_size: nil)
|
314
|
+
if transaction_open?
|
315
|
+
raise 'update_column_in_batches can not be run inside a transaction, ' \
|
316
|
+
'you can disable transactions by calling disable_ddl_transaction! ' \
|
317
|
+
'in the body of your migration class'
|
318
|
+
end
|
319
|
+
|
320
|
+
table = Arel::Table.new(table)
|
321
|
+
|
322
|
+
count_arel = table.project(Arel.star.count.as('count'))
|
323
|
+
count_arel = yield table, count_arel if block_given?
|
324
|
+
|
325
|
+
total = exec_query(count_arel.to_sql).to_hash.first['count'].to_i
|
326
|
+
|
327
|
+
return if total == 0
|
328
|
+
|
329
|
+
if batch_size.nil?
|
330
|
+
# Update in batches of 5% until we run out of any rows to update.
|
331
|
+
batch_size = ((total / 100.0) * 5.0).ceil
|
332
|
+
max_size = 1000
|
333
|
+
|
334
|
+
# The upper limit is 1000 to ensure we don't lock too many rows. For
|
335
|
+
# example, for "merge_requests" even 1% of the table is around 35 000
|
336
|
+
# rows for GitLab.com.
|
337
|
+
batch_size = max_size if batch_size > max_size
|
338
|
+
end
|
339
|
+
|
340
|
+
start_arel = table.project(table[:id]).order(table[:id].asc).take(1)
|
341
|
+
start_arel = yield table, start_arel if block_given?
|
342
|
+
start_id = exec_query(start_arel.to_sql).to_hash.first['id'].to_i
|
343
|
+
|
344
|
+
loop do
|
345
|
+
stop_arel = table.project(table[:id])
|
346
|
+
.where(table[:id].gteq(start_id))
|
347
|
+
.order(table[:id].asc)
|
348
|
+
.take(1)
|
349
|
+
.skip(batch_size)
|
350
|
+
|
351
|
+
stop_arel = yield table, stop_arel if block_given?
|
352
|
+
stop_row = exec_query(stop_arel.to_sql).to_hash.first
|
353
|
+
|
354
|
+
update_arel = Arel::UpdateManager.new
|
355
|
+
.table(table)
|
356
|
+
.set([[table[column], value]])
|
357
|
+
.where(table[:id].gteq(start_id))
|
358
|
+
|
359
|
+
if stop_row
|
360
|
+
stop_id = stop_row['id'].to_i
|
361
|
+
start_id = stop_id
|
362
|
+
update_arel = update_arel.where(table[:id].lt(stop_id))
|
363
|
+
end
|
364
|
+
|
365
|
+
update_arel = yield table, update_arel if block_given?
|
366
|
+
|
367
|
+
execute(update_arel.to_sql)
|
368
|
+
|
369
|
+
# There are no more rows left to update.
|
370
|
+
break unless stop_row
|
371
|
+
end
|
372
|
+
end
|
373
|
+
|
374
|
+
# Adds a column with a default value without locking an entire table.
|
375
|
+
#
|
376
|
+
# This method runs the following steps:
|
377
|
+
#
|
378
|
+
# 1. Add the column with a default value of NULL.
|
379
|
+
# 2. Change the default value of the column to the specified value.
|
380
|
+
# 3. Update all existing rows in batches.
|
381
|
+
# 4. Set a `NOT NULL` constraint on the column if desired (the default).
|
382
|
+
#
|
383
|
+
# These steps ensure a column can be added to a large and commonly used
|
384
|
+
# table without locking the entire table for the duration of the table
|
385
|
+
# modification.
|
386
|
+
#
|
387
|
+
# table - The name of the table to update.
|
388
|
+
# column - The name of the column to add.
|
389
|
+
# type - The column type (e.g. `:integer`).
|
390
|
+
# default - The default value for the column.
|
391
|
+
# limit - Sets a column limit. For example, for :integer, the default is
|
392
|
+
# 4-bytes. Set `limit: 8` to allow 8-byte integers.
|
393
|
+
# allow_null - When set to `true` the column will allow NULL values, the
|
394
|
+
# default is to not allow NULL values.
|
395
|
+
#
|
396
|
+
# This method can also take a block which is passed directly to the
|
397
|
+
# `update_column_in_batches` method.
|
398
|
+
def add_column_with_default(table, column, type, default:, limit: nil, allow_null: false, &block)
|
399
|
+
if transaction_open?
|
400
|
+
raise 'add_column_with_default can not be run inside a transaction, ' \
|
401
|
+
'you can disable transactions by calling disable_ddl_transaction! ' \
|
402
|
+
'in the body of your migration class'
|
403
|
+
end
|
404
|
+
|
405
|
+
disable_statement_timeout do
|
406
|
+
transaction do
|
407
|
+
if limit
|
408
|
+
add_column(table, column, type, default: nil, limit: limit)
|
409
|
+
else
|
410
|
+
add_column(table, column, type, default: nil)
|
411
|
+
end
|
412
|
+
|
413
|
+
# Changing the default before the update ensures any newly inserted
|
414
|
+
# rows already use the proper default value.
|
415
|
+
change_column_default(table, column, default)
|
416
|
+
end
|
417
|
+
|
418
|
+
begin
|
419
|
+
update_column_in_batches(table, column, default, &block)
|
420
|
+
|
421
|
+
change_column_null(table, column, false) unless allow_null
|
422
|
+
# We want to rescue _all_ exceptions here, even those that don't inherit
|
423
|
+
# from StandardError.
|
424
|
+
rescue Exception => error # rubocop: disable all
|
425
|
+
remove_column(table, column)
|
426
|
+
|
427
|
+
raise error
|
428
|
+
end
|
429
|
+
end
|
430
|
+
end
|
431
|
+
|
432
|
+
# Renames a column without requiring downtime.
|
433
|
+
#
|
434
|
+
# Concurrent renames work by using database triggers to ensure both the
|
435
|
+
# old and new column are in sync. However, this method will _not_ remove
|
436
|
+
# the triggers or the old column automatically; this needs to be done
|
437
|
+
# manually in a post-deployment migration. This can be done using the
|
438
|
+
# method `cleanup_concurrent_column_rename`.
|
439
|
+
#
|
440
|
+
# table - The name of the database table containing the column.
|
441
|
+
# old - The old column name.
|
442
|
+
# new - The new column name.
|
443
|
+
# type - The type of the new column. If no type is given the old column's
|
444
|
+
# type is used.
|
445
|
+
def rename_column_concurrently(table, old, new, type: nil)
|
446
|
+
raise 'rename_column_concurrently can not be run inside a transaction' if transaction_open?
|
447
|
+
|
448
|
+
check_trigger_permissions!(table)
|
449
|
+
|
450
|
+
old_col = column_for(table, old)
|
451
|
+
new_type = type || old_col.type
|
452
|
+
|
453
|
+
add_column(table, new, new_type,
|
454
|
+
limit: old_col.limit,
|
455
|
+
precision: old_col.precision,
|
456
|
+
scale: old_col.scale)
|
457
|
+
|
458
|
+
# We set the default value _after_ adding the column so we don't end up
|
459
|
+
# updating any existing data with the default value. This isn't
|
460
|
+
# necessary since we copy over old values further down.
|
461
|
+
change_column_default(table, new, old_col.default) if old_col.default
|
462
|
+
|
463
|
+
install_rename_triggers(table, old, new)
|
464
|
+
|
465
|
+
update_column_in_batches(table, new, Arel::Table.new(table)[old])
|
466
|
+
|
467
|
+
change_column_null(table, new, false) unless old_col.null
|
468
|
+
|
469
|
+
copy_indexes(table, old, new)
|
470
|
+
copy_foreign_keys(table, old, new)
|
471
|
+
end
|
472
|
+
|
473
|
+
# Installs triggers in a table that keep a new column in sync with an old
|
474
|
+
# one.
|
475
|
+
#
|
476
|
+
# table - The name of the table to install the trigger in.
|
477
|
+
# old_column - The name of the old column.
|
478
|
+
# new_column - The name of the new column.
|
479
|
+
def install_rename_triggers(table, old_column, new_column)
|
480
|
+
trigger_name = rename_trigger_name(table, old_column, new_column)
|
481
|
+
quoted_table = quote_table_name(table)
|
482
|
+
quoted_old = quote_column_name(old_column)
|
483
|
+
quoted_new = quote_column_name(new_column)
|
484
|
+
|
485
|
+
if Database.postgresql?
|
486
|
+
install_rename_triggers_for_postgresql(trigger_name, quoted_table,
|
487
|
+
quoted_old, quoted_new)
|
488
|
+
else
|
489
|
+
install_rename_triggers_for_mysql(trigger_name, quoted_table,
|
490
|
+
quoted_old, quoted_new)
|
491
|
+
end
|
492
|
+
end
|
493
|
+
|
494
|
+
# Changes the type of a column concurrently.
|
495
|
+
#
|
496
|
+
# table - The table containing the column.
|
497
|
+
# column - The name of the column to change.
|
498
|
+
# new_type - The new column type.
|
499
|
+
def change_column_type_concurrently(table, column, new_type)
|
500
|
+
temp_column = "#{column}_for_type_change"
|
501
|
+
|
502
|
+
rename_column_concurrently(table, column, temp_column, type: new_type)
|
503
|
+
end
|
504
|
+
|
505
|
+
# Performs cleanup of a concurrent type change.
|
506
|
+
#
|
507
|
+
# table - The table containing the column.
|
508
|
+
# column - The name of the column to change.
|
509
|
+
# new_type - The new column type.
|
510
|
+
def cleanup_concurrent_column_type_change(table, column)
|
511
|
+
temp_column = "#{column}_for_type_change"
|
512
|
+
|
513
|
+
transaction do
|
514
|
+
# This has to be performed in a transaction as otherwise we might have
|
515
|
+
# inconsistent data.
|
516
|
+
cleanup_concurrent_column_rename(table, column, temp_column)
|
517
|
+
rename_column(table, temp_column, column)
|
518
|
+
end
|
519
|
+
end
|
520
|
+
|
521
|
+
# Cleans up a concurrent column name.
|
522
|
+
#
|
523
|
+
# This method takes care of removing previously installed triggers as well
|
524
|
+
# as removing the old column.
|
525
|
+
#
|
526
|
+
# table - The name of the database table.
|
527
|
+
# old - The name of the old column.
|
528
|
+
# new - The name of the new column.
|
529
|
+
def cleanup_concurrent_column_rename(table, old, new)
|
530
|
+
trigger_name = rename_trigger_name(table, old, new)
|
531
|
+
|
532
|
+
check_trigger_permissions!(table)
|
533
|
+
|
534
|
+
if Database.postgresql?
|
535
|
+
remove_rename_triggers_for_postgresql(table, trigger_name)
|
536
|
+
else
|
537
|
+
remove_rename_triggers_for_mysql(trigger_name)
|
538
|
+
end
|
539
|
+
|
540
|
+
remove_column(table, old)
|
541
|
+
end
|
542
|
+
|
543
|
+
# TODO: resolve BackgroundMigrationWorker dependency in a clean way
|
544
|
+
|
545
|
+
# Changes the column type of a table using a background migration.
|
546
|
+
#
|
547
|
+
# Because this method uses a background migration it's more suitable for
|
548
|
+
# large tables. For small tables it's better to use
|
549
|
+
# `change_column_type_concurrently` since it can complete its work in a
|
550
|
+
# much shorter amount of time and doesn't rely on Sidekiq.
|
551
|
+
#
|
552
|
+
# Example usage:
|
553
|
+
#
|
554
|
+
# class Issue < ActiveRecord::Base
|
555
|
+
# self.table_name = 'issues'
|
556
|
+
#
|
557
|
+
# include EachBatch
|
558
|
+
#
|
559
|
+
# def self.to_migrate
|
560
|
+
# where('closed_at IS NOT NULL')
|
561
|
+
# end
|
562
|
+
# end
|
563
|
+
#
|
564
|
+
# change_column_type_using_background_migration(
|
565
|
+
# Issue.to_migrate,
|
566
|
+
# :closed_at,
|
567
|
+
# :datetime_with_timezone
|
568
|
+
# )
|
569
|
+
#
|
570
|
+
# Reverting a migration like this is done exactly the same way, just with
|
571
|
+
# a different type to migrate to (e.g. `:datetime` in the above example).
|
572
|
+
#
|
573
|
+
# relation - An ActiveRecord relation to use for scheduling jobs and
|
574
|
+
# figuring out what table we're modifying. This relation _must_
|
575
|
+
# have the EachBatch module included.
|
576
|
+
#
|
577
|
+
# column - The name of the column for which the type will be changed.
|
578
|
+
#
|
579
|
+
# new_type - The new type of the column.
|
580
|
+
#
|
581
|
+
# batch_size - The number of rows to schedule in a single background
|
582
|
+
# migration.
|
583
|
+
#
|
584
|
+
# interval - The time interval between every background migration.
|
585
|
+
# def change_column_type_using_background_migration(
|
586
|
+
# relation,
|
587
|
+
# column,
|
588
|
+
# new_type,
|
589
|
+
# batch_size: 10_000,
|
590
|
+
# interval: 10.minutes
|
591
|
+
# )
|
592
|
+
#
|
593
|
+
# unless relation.model < EachBatch
|
594
|
+
# raise TypeError, 'The relation must include the EachBatch module'
|
595
|
+
# end
|
596
|
+
#
|
597
|
+
# temp_column = "#{column}_for_type_change"
|
598
|
+
# table = relation.table_name
|
599
|
+
# max_index = 0
|
600
|
+
#
|
601
|
+
# add_column(table, temp_column, new_type)
|
602
|
+
# install_rename_triggers(table, column, temp_column)
|
603
|
+
#
|
604
|
+
# # Schedule the jobs that will copy the data from the old column to the
|
605
|
+
# # new one. Rows with NULL values in our source column are skipped since
|
606
|
+
# # the target column is already NULL at this point.
|
607
|
+
# relation.where.not(column => nil).each_batch(of: batch_size) do |batch, index|
|
608
|
+
# start_id, end_id = batch.pluck('MIN(id), MAX(id)').first
|
609
|
+
# max_index = index
|
610
|
+
#
|
611
|
+
# BackgroundMigrationWorker.perform_in(
|
612
|
+
# index * interval,
|
613
|
+
# 'CopyColumn',
|
614
|
+
# [table, column, temp_column, start_id, end_id]
|
615
|
+
# )
|
616
|
+
# end
|
617
|
+
#
|
618
|
+
# # Schedule the renaming of the column to happen (initially) 1 hour after
|
619
|
+
# # the last batch finished.
|
620
|
+
# BackgroundMigrationWorker.perform_in(
|
621
|
+
# (max_index * interval) + 1.hour,
|
622
|
+
# 'CleanupConcurrentTypeChange',
|
623
|
+
# [table, column, temp_column]
|
624
|
+
# )
|
625
|
+
#
|
626
|
+
# if perform_background_migration_inline?
|
627
|
+
# # To ensure the schema is up to date immediately we perform the
|
628
|
+
# # migration inline in dev / test environments.
|
629
|
+
# Gitlab::BackgroundMigration.steal('CopyColumn')
|
630
|
+
# Gitlab::BackgroundMigration.steal('CleanupConcurrentTypeChange')
|
631
|
+
# end
|
632
|
+
# end
|
633
|
+
|
634
|
+
# Renames a column using a background migration.
|
635
|
+
#
|
636
|
+
# Because this method uses a background migration it's more suitable for
|
637
|
+
# large tables. For small tables it's better to use
|
638
|
+
# `rename_column_concurrently` since it can complete its work in a much
|
639
|
+
# shorter amount of time and doesn't rely on Sidekiq.
|
640
|
+
#
|
641
|
+
# Example usage:
|
642
|
+
#
|
643
|
+
# rename_column_using_background_migration(
|
644
|
+
# :users,
|
645
|
+
# :feed_token,
|
646
|
+
# :rss_token
|
647
|
+
# )
|
648
|
+
#
|
649
|
+
# table - The name of the database table containing the column.
|
650
|
+
#
|
651
|
+
# old - The old column name.
|
652
|
+
#
|
653
|
+
# new - The new column name.
|
654
|
+
#
|
655
|
+
# type - The type of the new column. If no type is given the old column's
|
656
|
+
# type is used.
|
657
|
+
#
|
658
|
+
# batch_size - The number of rows to schedule in a single background
|
659
|
+
# migration.
|
660
|
+
#
|
661
|
+
# interval - The time interval between every background migration.
|
662
|
+
# def rename_column_using_background_migration(
|
663
|
+
# table,
|
664
|
+
# old_column,
|
665
|
+
# new_column,
|
666
|
+
# type: nil,
|
667
|
+
# batch_size: 10_000,
|
668
|
+
# interval: 10.minutes
|
669
|
+
# )
|
670
|
+
#
|
671
|
+
# check_trigger_permissions!(table)
|
672
|
+
#
|
673
|
+
# old_col = column_for(table, old_column)
|
674
|
+
# new_type = type || old_col.type
|
675
|
+
# max_index = 0
|
676
|
+
#
|
677
|
+
# add_column(table, new_column, new_type,
|
678
|
+
# limit: old_col.limit,
|
679
|
+
# precision: old_col.precision,
|
680
|
+
# scale: old_col.scale)
|
681
|
+
#
|
682
|
+
# # We set the default value _after_ adding the column so we don't end up
|
683
|
+
# # updating any existing data with the default value. This isn't
|
684
|
+
# # necessary since we copy over old values further down.
|
685
|
+
# change_column_default(table, new_column, old_col.default) if old_col.default
|
686
|
+
#
|
687
|
+
# install_rename_triggers(table, old_column, new_column)
|
688
|
+
#
|
689
|
+
# model = Class.new(ActiveRecord::Base) do
|
690
|
+
# self.table_name = table
|
691
|
+
#
|
692
|
+
# include ::EachBatch
|
693
|
+
# end
|
694
|
+
#
|
695
|
+
# # Schedule the jobs that will copy the data from the old column to the
|
696
|
+
# # new one. Rows with NULL values in our source column are skipped since
|
697
|
+
# # the target column is already NULL at this point.
|
698
|
+
# model.where.not(old_column => nil).each_batch(of: batch_size) do |batch, index|
|
699
|
+
# start_id, end_id = batch.pluck('MIN(id), MAX(id)').first
|
700
|
+
# max_index = index
|
701
|
+
#
|
702
|
+
# BackgroundMigrationWorker.perform_in(
|
703
|
+
# index * interval,
|
704
|
+
# 'CopyColumn',
|
705
|
+
# [table, old_column, new_column, start_id, end_id]
|
706
|
+
# )
|
707
|
+
# end
|
708
|
+
#
|
709
|
+
# # Schedule the renaming of the column to happen (initially) 1 hour after
|
710
|
+
# # the last batch finished.
|
711
|
+
# BackgroundMigrationWorker.perform_in(
|
712
|
+
# (max_index * interval) + 1.hour,
|
713
|
+
# 'CleanupConcurrentRename',
|
714
|
+
# [table, old_column, new_column]
|
715
|
+
# )
|
716
|
+
#
|
717
|
+
# if perform_background_migration_inline?
|
718
|
+
# # To ensure the schema is up to date immediately we perform the
|
719
|
+
# # migration inline in dev / test environments.
|
720
|
+
# Gitlab::BackgroundMigration.steal('CopyColumn')
|
721
|
+
# Gitlab::BackgroundMigration.steal('CleanupConcurrentRename')
|
722
|
+
# end
|
723
|
+
# end
|
724
|
+
#
|
725
|
+
# def perform_background_migration_inline?
|
726
|
+
# Rails.env.test? || Rails.env.development?
|
727
|
+
# end
|
728
|
+
|
729
|
+
# Performs a concurrent column rename when using PostgreSQL.
|
730
|
+
def install_rename_triggers_for_postgresql(trigger, table, old, new)
|
731
|
+
execute <<-EOF.strip_heredoc
|
732
|
+
CREATE OR REPLACE FUNCTION #{trigger}()
|
733
|
+
RETURNS trigger AS
|
734
|
+
$BODY$
|
735
|
+
BEGIN
|
736
|
+
NEW.#{new} := NEW.#{old};
|
737
|
+
RETURN NEW;
|
738
|
+
END;
|
739
|
+
$BODY$
|
740
|
+
LANGUAGE 'plpgsql'
|
741
|
+
VOLATILE
|
742
|
+
EOF
|
743
|
+
|
744
|
+
execute <<-EOF.strip_heredoc
|
745
|
+
CREATE TRIGGER #{trigger}
|
746
|
+
BEFORE INSERT OR UPDATE
|
747
|
+
ON #{table}
|
748
|
+
FOR EACH ROW
|
749
|
+
EXECUTE PROCEDURE #{trigger}()
|
750
|
+
EOF
|
751
|
+
end
|
752
|
+
|
753
|
+
# Installs the triggers necessary to perform a concurrent column rename on
|
754
|
+
# MySQL.
|
755
|
+
def install_rename_triggers_for_mysql(trigger, table, old, new)
|
756
|
+
execute <<-EOF.strip_heredoc
|
757
|
+
CREATE TRIGGER #{trigger}_insert
|
758
|
+
BEFORE INSERT
|
759
|
+
ON #{table}
|
760
|
+
FOR EACH ROW
|
761
|
+
SET NEW.#{new} = NEW.#{old}
|
762
|
+
EOF
|
763
|
+
|
764
|
+
execute <<-EOF.strip_heredoc
|
765
|
+
CREATE TRIGGER #{trigger}_update
|
766
|
+
BEFORE UPDATE
|
767
|
+
ON #{table}
|
768
|
+
FOR EACH ROW
|
769
|
+
SET NEW.#{new} = NEW.#{old}
|
770
|
+
EOF
|
771
|
+
end
|
772
|
+
|
773
|
+
# Removes the triggers used for renaming a PostgreSQL column concurrently.
|
774
|
+
def remove_rename_triggers_for_postgresql(table, trigger)
|
775
|
+
execute("DROP TRIGGER IF EXISTS #{trigger} ON #{table}")
|
776
|
+
execute("DROP FUNCTION IF EXISTS #{trigger}()")
|
777
|
+
end
|
778
|
+
|
779
|
+
# Removes the triggers used for renaming a MySQL column concurrently.
|
780
|
+
def remove_rename_triggers_for_mysql(trigger)
|
781
|
+
execute("DROP TRIGGER IF EXISTS #{trigger}_insert")
|
782
|
+
execute("DROP TRIGGER IF EXISTS #{trigger}_update")
|
783
|
+
end
|
784
|
+
|
785
|
+
# Returns the (base) name to use for triggers when renaming columns.
|
786
|
+
def rename_trigger_name(table, old, new)
|
787
|
+
'trigger_' + Digest::SHA256.hexdigest("#{table}_#{old}_#{new}").first(12)
|
788
|
+
end
|
789
|
+
|
790
|
+
# Returns an Array containing the indexes for the given column
|
791
|
+
def indexes_for(table, column)
|
792
|
+
column = column.to_s
|
793
|
+
|
794
|
+
indexes(table).select { |index| index.columns.include?(column) }
|
795
|
+
end
|
796
|
+
|
797
|
+
# Returns an Array containing the foreign keys for the given column.
|
798
|
+
def foreign_keys_for(table, column)
|
799
|
+
column = column.to_s
|
800
|
+
|
801
|
+
foreign_keys(table).select { |fk| fk.column == column }
|
802
|
+
end
|
803
|
+
|
804
|
+
# Copies all indexes for the old column to a new column.
|
805
|
+
#
|
806
|
+
# table - The table containing the columns and indexes.
|
807
|
+
# old - The old column.
|
808
|
+
# new - The new column.
|
809
|
+
def copy_indexes(table, old, new)
|
810
|
+
old = old.to_s
|
811
|
+
new = new.to_s
|
812
|
+
|
813
|
+
indexes_for(table, old).each do |index|
|
814
|
+
new_columns = index.columns.map do |column|
|
815
|
+
column == old ? new : column
|
816
|
+
end
|
817
|
+
|
818
|
+
# This is necessary as we can't properly rename indexes such as
|
819
|
+
# "ci_taggings_idx".
|
820
|
+
unless index.name.include?(old)
|
821
|
+
raise "The index #{index.name} can not be copied as it does not "\
|
822
|
+
'mention the old column. You have to rename this index manually first.'
|
823
|
+
end
|
824
|
+
|
825
|
+
name = index.name.gsub(old, new)
|
826
|
+
|
827
|
+
options = {
|
828
|
+
unique: index.unique,
|
829
|
+
name: name,
|
830
|
+
length: index.lengths,
|
831
|
+
order: index.orders
|
832
|
+
}
|
833
|
+
|
834
|
+
# These options are not supported by MySQL, so we only add them if
|
835
|
+
# they were previously set.
|
836
|
+
options[:using] = index.using if index.using
|
837
|
+
options[:where] = index.where if index.where
|
838
|
+
|
839
|
+
if index.opclasses.present?
|
840
|
+
opclasses = index.opclasses.dup
|
841
|
+
|
842
|
+
# Copy the operator classes for the old column (if any) to the new
|
843
|
+
# column.
|
844
|
+
opclasses[new] = opclasses.delete(old) if opclasses[old]
|
845
|
+
|
846
|
+
options[:opclasses] = opclasses
|
847
|
+
end
|
848
|
+
|
849
|
+
add_concurrent_index(table, new_columns, options)
|
850
|
+
end
|
851
|
+
end
|
852
|
+
|
853
|
+
# Copies all foreign keys for the old column to the new column.
|
854
|
+
#
|
855
|
+
# table - The table containing the columns and indexes.
|
856
|
+
# old - The old column.
|
857
|
+
# new - The new column.
|
858
|
+
def copy_foreign_keys(table, old, new)
|
859
|
+
foreign_keys_for(table, old).each do |fk|
|
860
|
+
add_concurrent_foreign_key(fk.from_table,
|
861
|
+
fk.to_table,
|
862
|
+
column: new,
|
863
|
+
on_delete: fk.on_delete)
|
864
|
+
end
|
865
|
+
end
|
866
|
+
|
867
|
+
# Returns the column for the given table and column name.
|
868
|
+
def column_for(table, name)
|
869
|
+
name = name.to_s
|
870
|
+
|
871
|
+
columns(table).find { |column| column.name == name }
|
872
|
+
end
|
873
|
+
|
874
|
+
# This will replace the first occurrence of a string in a column with
|
875
|
+
# the replacement
|
876
|
+
# On postgresql we can use `regexp_replace` for that.
|
877
|
+
# On mysql we find the location of the pattern, and overwrite it
|
878
|
+
# with the replacement
|
879
|
+
def replace_sql(column, pattern, replacement)
|
880
|
+
quoted_pattern = Arel::Nodes::Quoted.new(pattern.to_s)
|
881
|
+
quoted_replacement = Arel::Nodes::Quoted.new(replacement.to_s)
|
882
|
+
|
883
|
+
if Database.mysql?
|
884
|
+
locate = Arel::Nodes::NamedFunction
|
885
|
+
.new('locate', [quoted_pattern, column])
|
886
|
+
insert_in_place = Arel::Nodes::NamedFunction
|
887
|
+
.new('insert', [column, locate, pattern.size, quoted_replacement])
|
888
|
+
|
889
|
+
Arel::Nodes::SqlLiteral.new(insert_in_place.to_sql)
|
890
|
+
else
|
891
|
+
replace = Arel::Nodes::NamedFunction
|
892
|
+
.new('regexp_replace', [column, quoted_pattern, quoted_replacement])
|
893
|
+
Arel::Nodes::SqlLiteral.new(replace.to_sql)
|
894
|
+
end
|
895
|
+
end
|
896
|
+
|
897
|
+
def remove_foreign_key_if_exists(*args)
|
898
|
+
remove_foreign_key(*args) if foreign_key_exists?(*args)
|
899
|
+
end
|
900
|
+
|
901
|
+
def remove_foreign_key_without_error(*args)
|
902
|
+
remove_foreign_key(*args)
|
903
|
+
rescue ArgumentError
|
904
|
+
end
|
905
|
+
|
906
|
+
def sidekiq_queue_migrate(queue_from, to:)
|
907
|
+
while sidekiq_queue_length(queue_from) > 0
|
908
|
+
Sidekiq.redis do |conn|
|
909
|
+
conn.rpoplpush "queue:#{queue_from}", "queue:#{to}"
|
910
|
+
end
|
911
|
+
end
|
912
|
+
end
|
913
|
+
|
914
|
+
def sidekiq_queue_length(queue_name)
|
915
|
+
Sidekiq.redis do |conn|
|
916
|
+
conn.llen("queue:#{queue_name}")
|
917
|
+
end
|
918
|
+
end
|
919
|
+
|
920
|
+
def check_trigger_permissions!(table)
|
921
|
+
unless Grant.create_and_execute_trigger?(table)
|
922
|
+
dbname = Database.database_name
|
923
|
+
user = Database.username
|
924
|
+
|
925
|
+
raise <<~EOF
|
926
|
+
Your database user is not allowed to create, drop, or execute triggers on the
|
927
|
+
table #{table}.
|
928
|
+
|
929
|
+
If you are using PostgreSQL you can solve this by logging in to the GitLab
|
930
|
+
database (#{dbname}) using a super user and running:
|
931
|
+
|
932
|
+
ALTER #{user} WITH SUPERUSER
|
933
|
+
|
934
|
+
For MySQL you instead need to run:
|
935
|
+
|
936
|
+
GRANT ALL PRIVILEGES ON #{dbname}.* TO #{user}@'%'
|
937
|
+
|
938
|
+
Both queries will grant the user super user permissions, ensuring you don't run
|
939
|
+
into similar problems in the future (e.g. when new tables are created).
|
940
|
+
EOF
|
941
|
+
end
|
942
|
+
end
|
943
|
+
|
944
|
+
# Bulk queues background migration jobs for an entire table, batched by ID range.
|
945
|
+
# "Bulk" meaning many jobs will be pushed at a time for efficiency.
|
946
|
+
# If you need a delay interval per job, then use `queue_background_migration_jobs_by_range_at_intervals`.
|
947
|
+
#
|
948
|
+
# model_class - The table being iterated over
|
949
|
+
# job_class_name - The background migration job class as a string
|
950
|
+
# batch_size - The maximum number of rows per job
|
951
|
+
#
|
952
|
+
# Example:
|
953
|
+
#
|
954
|
+
# class Route < ActiveRecord::Base
|
955
|
+
# include EachBatch
|
956
|
+
# self.table_name = 'routes'
|
957
|
+
# end
|
958
|
+
#
|
959
|
+
# bulk_queue_background_migration_jobs_by_range(Route, 'ProcessRoutes')
|
960
|
+
#
|
961
|
+
# Where the model_class includes EachBatch, and the background migration exists:
|
962
|
+
#
|
963
|
+
# class Gitlab::BackgroundMigration::ProcessRoutes
|
964
|
+
# def perform(start_id, end_id)
|
965
|
+
# # do something
|
966
|
+
# end
|
967
|
+
# end
|
968
|
+
# def bulk_queue_background_migration_jobs_by_range(model_class, job_class_name, batch_size: BACKGROUND_MIGRATION_BATCH_SIZE)
|
969
|
+
# raise "#{model_class} does not have an ID to use for batch ranges" unless model_class.column_names.include?('id')
|
970
|
+
#
|
971
|
+
# jobs = []
|
972
|
+
# table_name = model_class.quoted_table_name
|
973
|
+
#
|
974
|
+
# model_class.each_batch(of: batch_size) do |relation|
|
975
|
+
# start_id, end_id = relation.pluck("MIN(#{table_name}.id), MAX(#{table_name}.id)").first
|
976
|
+
#
|
977
|
+
# if jobs.length >= BACKGROUND_MIGRATION_JOB_BUFFER_SIZE
|
978
|
+
# # Note: This code path generally only helps with many millions of rows
|
979
|
+
# # We push multiple jobs at a time to reduce the time spent in
|
980
|
+
# # Sidekiq/Redis operations. We're using this buffer based approach so we
|
981
|
+
# # don't need to run additional queries for every range.
|
982
|
+
# BackgroundMigrationWorker.bulk_perform_async(jobs)
|
983
|
+
# jobs.clear
|
984
|
+
# end
|
985
|
+
#
|
986
|
+
# jobs << [job_class_name, [start_id, end_id]]
|
987
|
+
# end
|
988
|
+
#
|
989
|
+
# BackgroundMigrationWorker.bulk_perform_async(jobs) unless jobs.empty?
|
990
|
+
# end
|
991
|
+
|
992
|
+
# Queues background migration jobs for an entire table, batched by ID range.
|
993
|
+
# Each job is scheduled with a `delay_interval` in between.
|
994
|
+
# If you use a small interval, then some jobs may run at the same time.
|
995
|
+
#
|
996
|
+
# model_class - The table or relation being iterated over
|
997
|
+
# job_class_name - The background migration job class as a string
|
998
|
+
# delay_interval - The duration between each job's scheduled time (must respond to `to_f`)
|
999
|
+
# batch_size - The maximum number of rows per job
|
1000
|
+
#
|
1001
|
+
# Example:
|
1002
|
+
#
|
1003
|
+
# class Route < ActiveRecord::Base
|
1004
|
+
# include EachBatch
|
1005
|
+
# self.table_name = 'routes'
|
1006
|
+
# end
|
1007
|
+
#
|
1008
|
+
# queue_background_migration_jobs_by_range_at_intervals(Route, 'ProcessRoutes', 1.minute)
|
1009
|
+
#
|
1010
|
+
# Where the model_class includes EachBatch, and the background migration exists:
|
1011
|
+
#
|
1012
|
+
# class Gitlab::BackgroundMigration::ProcessRoutes
|
1013
|
+
# def perform(start_id, end_id)
|
1014
|
+
# # do something
|
1015
|
+
# end
|
1016
|
+
# end
|
1017
|
+
# def queue_background_migration_jobs_by_range_at_intervals(model_class, job_class_name, delay_interval, batch_size: BACKGROUND_MIGRATION_BATCH_SIZE)
|
1018
|
+
# raise "#{model_class} does not have an ID to use for batch ranges" unless model_class.column_names.include?('id')
|
1019
|
+
#
|
1020
|
+
# # To not overload the worker too much we enforce a minimum interval both
|
1021
|
+
# # when scheduling and performing jobs.
|
1022
|
+
# if delay_interval < BackgroundMigrationWorker.minimum_interval
|
1023
|
+
# delay_interval = BackgroundMigrationWorker.minimum_interval
|
1024
|
+
# end
|
1025
|
+
#
|
1026
|
+
# model_class.each_batch(of: batch_size) do |relation, index|
|
1027
|
+
# start_id, end_id = relation.pluck('MIN(id), MAX(id)').first
|
1028
|
+
#
|
1029
|
+
# # `BackgroundMigrationWorker.bulk_perform_in` schedules all jobs for
|
1030
|
+
# # the same time, which is not helpful in most cases where we wish to
|
1031
|
+
# # spread the work over time.
|
1032
|
+
# BackgroundMigrationWorker.perform_in(delay_interval * index, job_class_name, [start_id, end_id])
|
1033
|
+
# end
|
1034
|
+
# end
|
1035
|
+
|
1036
|
+
# Fetches indexes on a column by name for postgres.
|
1037
|
+
#
|
1038
|
+
# This will include indexes using an expression on the column, for example:
|
1039
|
+
# `CREATE INDEX CONCURRENTLY index_name ON table (LOWER(column));`
|
1040
|
+
#
|
1041
|
+
# For mysql, it falls back to the default ActiveRecord implementation that
|
1042
|
+
# will not find custom indexes. But it will select by name without passing
|
1043
|
+
# a column.
|
1044
|
+
#
|
1045
|
+
# We can remove this when upgrading to Rails 5 with an updated `index_exists?`:
|
1046
|
+
# - https://github.com/rails/rails/commit/edc2b7718725016e988089b5fb6d6fb9d6e16882
|
1047
|
+
#
|
1048
|
+
# Or this can be removed when we no longer support postgres < 9.5, so we
|
1049
|
+
# can use `CREATE INDEX IF NOT EXISTS`.
|
1050
|
+
def index_exists_by_name?(table, index)
|
1051
|
+
# We can't fall back to the normal `index_exists?` method because that
|
1052
|
+
# does not find indexes without passing a column name.
|
1053
|
+
if indexes(table).map(&:name).include?(index.to_s)
|
1054
|
+
true
|
1055
|
+
elsif Database.postgresql?
|
1056
|
+
postgres_exists_by_name?(table, index)
|
1057
|
+
else
|
1058
|
+
false
|
1059
|
+
end
|
1060
|
+
end
|
1061
|
+
|
1062
|
+
def postgres_exists_by_name?(table, name)
|
1063
|
+
index_sql = <<~SQL
|
1064
|
+
SELECT COUNT(*)
|
1065
|
+
FROM pg_index
|
1066
|
+
JOIN pg_class i ON (indexrelid=i.oid)
|
1067
|
+
JOIN pg_class t ON (indrelid=t.oid)
|
1068
|
+
WHERE i.relname = '#{name}' AND t.relname = '#{table}'
|
1069
|
+
SQL
|
1070
|
+
|
1071
|
+
connection.select_value(index_sql).to_i > 0
|
1072
|
+
end
|
1073
|
+
|
1074
|
+
def mysql_compatible_index_length
|
1075
|
+
Database.mysql? ? 20 : nil
|
1076
|
+
end
|
1077
|
+
end
|
1078
|
+
end
|
@@ -0,0 +1,12 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require 'migration_patterns/database_helpers'
|
4
|
+
require 'migration_patterns/migration_helpers'
|
5
|
+
# require 'migration_patterns/errors'
|
6
|
+
|
7
|
+
# require 'date'
|
8
|
+
# require 'json'
|
9
|
+
# require 'pathname'
|
10
|
+
|
11
|
+
module MigrationPatterns
|
12
|
+
end
|
metadata
ADDED
@@ -0,0 +1,134 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: migration-patterns
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 0.0.0.pre.rc1
|
5
|
+
platform: ruby
|
6
|
+
authors:
|
7
|
+
- Adam David
|
8
|
+
- Gitlab
|
9
|
+
autorequire:
|
10
|
+
bindir: bin
|
11
|
+
cert_chain: []
|
12
|
+
date: 2019-06-17 00:00:00.000000000 Z
|
13
|
+
dependencies:
|
14
|
+
- !ruby/object:Gem::Dependency
|
15
|
+
name: bundler
|
16
|
+
requirement: !ruby/object:Gem::Requirement
|
17
|
+
requirements:
|
18
|
+
- - ">="
|
19
|
+
- !ruby/object:Gem::Version
|
20
|
+
version: '0'
|
21
|
+
type: :development
|
22
|
+
prerelease: false
|
23
|
+
version_requirements: !ruby/object:Gem::Requirement
|
24
|
+
requirements:
|
25
|
+
- - ">="
|
26
|
+
- !ruby/object:Gem::Version
|
27
|
+
version: '0'
|
28
|
+
- !ruby/object:Gem::Dependency
|
29
|
+
name: pry
|
30
|
+
requirement: !ruby/object:Gem::Requirement
|
31
|
+
requirements:
|
32
|
+
- - ">="
|
33
|
+
- !ruby/object:Gem::Version
|
34
|
+
version: '0'
|
35
|
+
type: :development
|
36
|
+
prerelease: false
|
37
|
+
version_requirements: !ruby/object:Gem::Requirement
|
38
|
+
requirements:
|
39
|
+
- - ">="
|
40
|
+
- !ruby/object:Gem::Version
|
41
|
+
version: '0'
|
42
|
+
- !ruby/object:Gem::Dependency
|
43
|
+
name: rake
|
44
|
+
requirement: !ruby/object:Gem::Requirement
|
45
|
+
requirements:
|
46
|
+
- - ">="
|
47
|
+
- !ruby/object:Gem::Version
|
48
|
+
version: '0'
|
49
|
+
type: :development
|
50
|
+
prerelease: false
|
51
|
+
version_requirements: !ruby/object:Gem::Requirement
|
52
|
+
requirements:
|
53
|
+
- - ">="
|
54
|
+
- !ruby/object:Gem::Version
|
55
|
+
version: '0'
|
56
|
+
- !ruby/object:Gem::Dependency
|
57
|
+
name: rspec
|
58
|
+
requirement: !ruby/object:Gem::Requirement
|
59
|
+
requirements:
|
60
|
+
- - ">="
|
61
|
+
- !ruby/object:Gem::Version
|
62
|
+
version: '0'
|
63
|
+
type: :development
|
64
|
+
prerelease: false
|
65
|
+
version_requirements: !ruby/object:Gem::Requirement
|
66
|
+
requirements:
|
67
|
+
- - ">="
|
68
|
+
- !ruby/object:Gem::Version
|
69
|
+
version: '0'
|
70
|
+
- !ruby/object:Gem::Dependency
|
71
|
+
name: rubocop
|
72
|
+
requirement: !ruby/object:Gem::Requirement
|
73
|
+
requirements:
|
74
|
+
- - ">="
|
75
|
+
- !ruby/object:Gem::Version
|
76
|
+
version: '0'
|
77
|
+
type: :development
|
78
|
+
prerelease: false
|
79
|
+
version_requirements: !ruby/object:Gem::Requirement
|
80
|
+
requirements:
|
81
|
+
- - ">="
|
82
|
+
- !ruby/object:Gem::Version
|
83
|
+
version: '0'
|
84
|
+
- !ruby/object:Gem::Dependency
|
85
|
+
name: rubocop-rails
|
86
|
+
requirement: !ruby/object:Gem::Requirement
|
87
|
+
requirements:
|
88
|
+
- - ">="
|
89
|
+
- !ruby/object:Gem::Version
|
90
|
+
version: '0'
|
91
|
+
type: :development
|
92
|
+
prerelease: false
|
93
|
+
version_requirements: !ruby/object:Gem::Requirement
|
94
|
+
requirements:
|
95
|
+
- - ">="
|
96
|
+
- !ruby/object:Gem::Version
|
97
|
+
version: '0'
|
98
|
+
description:
|
99
|
+
email:
|
100
|
+
- adamrdavid@gmail.com
|
101
|
+
executables: []
|
102
|
+
extensions: []
|
103
|
+
extra_rdoc_files: []
|
104
|
+
files:
|
105
|
+
- lib/migration_patterns.rb
|
106
|
+
- lib/migration_patterns/background_job.rb
|
107
|
+
- lib/migration_patterns/database_helpers.rb
|
108
|
+
- lib/migration_patterns/migration_helpers.rb
|
109
|
+
- lib/migration_patterns/version.rb
|
110
|
+
homepage: https://github.com/adamrdavid/migration-patterns
|
111
|
+
licenses:
|
112
|
+
- MIT
|
113
|
+
metadata: {}
|
114
|
+
post_install_message:
|
115
|
+
rdoc_options: []
|
116
|
+
require_paths:
|
117
|
+
- lib
|
118
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
119
|
+
requirements:
|
120
|
+
- - ">="
|
121
|
+
- !ruby/object:Gem::Version
|
122
|
+
version: '0'
|
123
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
124
|
+
requirements:
|
125
|
+
- - ">"
|
126
|
+
- !ruby/object:Gem::Version
|
127
|
+
version: 1.3.1
|
128
|
+
requirements: []
|
129
|
+
rubyforge_project:
|
130
|
+
rubygems_version: 2.7.6
|
131
|
+
signing_key:
|
132
|
+
specification_version: 4
|
133
|
+
summary: Zero downtime migration helpers for rails
|
134
|
+
test_files: []
|