strong_migrations 0.6.8 → 1.6.1

Sign up to get free protection for your applications and to get access to all the features.
@@ -1,13 +1,17 @@
1
1
  module StrongMigrations
2
2
  class Checker
3
- attr_accessor :direction
3
+ include Checks
4
+ include SafeMethods
5
+
6
+ attr_accessor :direction, :transaction_disabled, :timeouts_set
4
7
 
5
8
  def initialize(migration)
6
9
  @migration = migration
7
10
  @new_tables = []
11
+ @new_columns = []
8
12
  @safe = false
9
13
  @timeouts_set = false
10
- @lock_timeout_checked = false
14
+ @committed = false
11
15
  end
12
16
 
13
17
  def safety_assured
@@ -21,464 +25,182 @@ module StrongMigrations
21
25
  end
22
26
 
23
27
  def perform(method, *args)
28
+ check_version_supported
24
29
  set_timeouts
25
30
  check_lock_timeout
26
31
 
27
- unless safe?
32
+ if !safe? || safe_by_default_method?(method)
33
+ # TODO better pattern
34
+ # see checks.rb for methods
28
35
  case method
29
- when :remove_column, :remove_columns, :remove_timestamps, :remove_reference, :remove_belongs_to
30
- columns =
31
- case method
32
- when :remove_timestamps
33
- ["created_at", "updated_at"]
34
- when :remove_column
35
- [args[1].to_s]
36
- when :remove_columns
37
- args[1..-1].map(&:to_s)
38
- else
39
- options = args[2] || {}
40
- reference = args[1]
41
- cols = []
42
- cols << "#{reference}_type" if options[:polymorphic]
43
- cols << "#{reference}_id"
44
- cols
45
- end
46
-
47
- code = "self.ignored_columns = #{columns.inspect}"
48
-
49
- raise_error :remove_column,
50
- model: args[0].to_s.classify,
51
- code: code,
52
- command: command_str(method, args),
53
- column_suffix: columns.size > 1 ? "s" : ""
54
- when :change_table
55
- raise_error :change_table, header: "Possibly dangerous operation"
56
- when :rename_table
57
- raise_error :rename_table
58
- when :rename_column
59
- raise_error :rename_column
60
- when :add_index
61
- table, columns, options = args
62
- options ||= {}
63
-
64
- if columns.is_a?(Array) && columns.size > 3 && !options[:unique]
65
- raise_error :add_index_columns, header: "Best practice"
66
- end
67
- if postgresql? && options[:algorithm] != :concurrently && !new_table?(table)
68
- raise_error :add_index, command: command_str("add_index", [table, columns, options.merge(algorithm: :concurrently)])
69
- end
70
- when :remove_index
71
- table, options = args
72
- unless options.is_a?(Hash)
73
- options = {column: options}
74
- end
75
- options ||= {}
76
-
77
- if postgresql? && options[:algorithm] != :concurrently && !new_table?(table)
78
- raise_error :remove_index, command: command_str("remove_index", [table, options.merge(algorithm: :concurrently)])
79
- end
36
+ when :add_check_constraint
37
+ check_add_check_constraint(*args)
80
38
  when :add_column
81
- table, column, type, options = args
82
- options ||= {}
83
- default = options[:default]
84
-
85
- if !default.nil? && !((postgresql? && postgresql_version >= Gem::Version.new("11")) || (mysql? && mysql_version >= Gem::Version.new("8.0.12")) || (mariadb? && mariadb_version >= Gem::Version.new("10.3.2")))
86
-
87
- if options[:null] == false
88
- options = options.except(:null)
89
- append = "
90
-
91
- Then add the NOT NULL constraint in separate migrations."
92
- end
93
-
94
- raise_error :add_column_default,
95
- add_command: command_str("add_column", [table, column, type, options.except(:default)]),
96
- change_command: command_str("change_column_default", [table, column, default]),
97
- remove_command: command_str("remove_column", [table, column]),
98
- code: backfill_code(table, column, default),
99
- append: append
100
- end
101
-
102
- if type.to_s == "json" && postgresql?
103
- raise_error :add_column_json
104
- end
39
+ check_add_column(*args)
40
+ when :add_exclusion_constraint
41
+ check_add_exclusion_constraint(*args)
42
+ when :add_foreign_key
43
+ check_add_foreign_key(*args)
44
+ when :add_index
45
+ check_add_index(*args)
46
+ when :add_reference, :add_belongs_to
47
+ check_add_reference(method, *args)
105
48
  when :change_column
106
- table, column, type, options = args
107
- options ||= {}
108
-
109
- safe = false
110
- existing_column = connection.columns(table).find { |c| c.name.to_s == column.to_s }
111
- if existing_column
112
- sql_type = existing_column.sql_type.split("(").first
113
- if postgresql?
114
- case type.to_s
115
- when "string", "text"
116
- # safe to change limit for varchar
117
- safe = ["character varying", "text"].include?(sql_type)
118
- when "numeric", "decimal"
119
- # numeric and decimal are equivalent and can be used interchangably
120
- safe = ["numeric", "decimal"].include?(sql_type) &&
121
- (
122
- (
123
- # unconstrained
124
- !options[:precision] && !options[:scale]
125
- ) || (
126
- # increased precision, same scale
127
- options[:precision] && existing_column.precision &&
128
- options[:precision] >= existing_column.precision &&
129
- options[:scale] == existing_column.scale
130
- )
131
- )
132
- when "datetime", "timestamp", "timestamptz"
133
- safe = ["timestamp without time zone", "timestamp with time zone"].include?(sql_type) &&
134
- postgresql_version >= Gem::Version.new("12") &&
135
- connection.select_all("SHOW timezone").first["TimeZone"] == "UTC"
136
- end
137
- elsif mysql? || mariadb?
138
- case type.to_s
139
- when "string"
140
- # https://dev.mysql.com/doc/refman/5.7/en/innodb-online-ddl-operations.html
141
- # https://mariadb.com/kb/en/innodb-online-ddl-operations-with-the-instant-alter-algorithm/#changing-the-data-type-of-a-column
142
- # increased limit, but doesn't change number of length bytes
143
- # 1-255 = 1 byte, 256-65532 = 2 bytes, 65533+ = too big for varchar
144
- limit = options[:limit] || 255
145
- safe = ["varchar"].include?(sql_type) &&
146
- limit >= existing_column.limit &&
147
- (limit <= 255 || existing_column.limit > 255)
148
- end
149
- end
150
- end
151
- raise_error :change_column unless safe
49
+ check_change_column(*args)
50
+ when :change_column_default
51
+ check_change_column_default(*args)
52
+ when :change_column_null
53
+ check_change_column_null(*args)
54
+ when :change_table
55
+ check_change_table
56
+ when :create_join_table
57
+ check_create_join_table(*args)
152
58
  when :create_table
153
- table, options = args
154
- options ||= {}
155
-
156
- raise_error :create_table if options[:force]
157
-
158
- # keep track of new tables of add_index check
159
- @new_tables << table.to_s
160
- when :add_reference, :add_belongs_to
161
- table, reference, options = args
162
- options ||= {}
163
-
164
- if postgresql?
165
- index_value = options.fetch(:index, true)
166
- concurrently_set = index_value.is_a?(Hash) && index_value[:algorithm] == :concurrently
167
- bad_index = index_value && !concurrently_set
168
-
169
- if bad_index || options[:foreign_key]
170
- columns = options[:polymorphic] ? [:"#{reference}_type", :"#{reference}_id"] : :"#{reference}_id"
171
-
172
- if index_value.is_a?(Hash)
173
- options[:index] = options[:index].merge(algorithm: :concurrently)
174
- else
175
- options = options.merge(index: {algorithm: :concurrently})
176
- end
177
-
178
- if options.delete(:foreign_key)
179
- headline = "Adding a validated foreign key locks the table."
180
- append = "
181
-
182
- Then add the foreign key in separate migrations."
183
- else
184
- headline = "Adding an index non-concurrently locks the table."
185
- end
186
-
187
- raise_error :add_reference,
188
- headline: headline,
189
- command: command_str(method, [table, reference, options]),
190
- append: append
191
- end
192
- end
59
+ check_create_table(*args)
193
60
  when :execute
194
- raise_error :execute, header: "Possibly dangerous operation"
195
- when :change_column_null
196
- table, column, null, default = args
197
- if !null
198
- if postgresql?
199
- safe = false
200
- if postgresql_version >= Gem::Version.new("12")
201
- # TODO likely need to quote the column in some situations
202
- safe = constraints(table).any? { |c| c["def"] == "CHECK ((#{column} IS NOT NULL))" }
203
- end
204
-
205
- unless safe
206
- # match https://github.com/nullobject/rein
207
- constraint_name = "#{table}_#{column}_null"
208
-
209
- validate_constraint_code = String.new(constraint_str("ALTER TABLE %s VALIDATE CONSTRAINT %s", [table, constraint_name]))
210
- if postgresql_version >= Gem::Version.new("12")
211
- validate_constraint_code << "\n #{command_str(:change_column_null, [table, column, null])}"
212
- validate_constraint_code << "\n #{constraint_str("ALTER TABLE %s DROP CONSTRAINT %s", [table, constraint_name])}"
213
- end
214
-
215
- raise_error :change_column_null_postgresql,
216
- add_constraint_code: constraint_str("ALTER TABLE %s ADD CONSTRAINT %s CHECK (%s IS NOT NULL) NOT VALID", [table, constraint_name, column]),
217
- validate_constraint_code: validate_constraint_code
218
- end
219
- elsif mysql? || mariadb?
220
- raise_error :change_column_null_mysql
221
- elsif !default.nil?
222
- raise_error :change_column_null,
223
- code: backfill_code(table, column, default)
224
- end
225
- end
226
- when :add_foreign_key
227
- from_table, to_table, options = args
228
- options ||= {}
229
-
230
- # always validated before 5.2
231
- validate = options.fetch(:validate, true) || ActiveRecord::VERSION::STRING < "5.2"
232
-
233
- if postgresql? && validate
234
- if ActiveRecord::VERSION::STRING < "5.2"
235
- # fk name logic from rails
236
- primary_key = options[:primary_key] || "id"
237
- column = options[:column] || "#{to_table.to_s.singularize}_id"
238
- hashed_identifier = Digest::SHA256.hexdigest("#{from_table}_#{column}_fk").first(10)
239
- fk_name = options[:name] || "fk_rails_#{hashed_identifier}"
240
-
241
- raise_error :add_foreign_key,
242
- add_foreign_key_code: constraint_str("ALTER TABLE %s ADD CONSTRAINT %s FOREIGN KEY (%s) REFERENCES %s (%s) NOT VALID", [from_table, fk_name, column, to_table, primary_key]),
243
- validate_foreign_key_code: constraint_str("ALTER TABLE %s VALIDATE CONSTRAINT %s", [from_table, fk_name])
244
- else
245
- raise_error :add_foreign_key,
246
- add_foreign_key_code: command_str("add_foreign_key", [from_table, to_table, options.merge(validate: false)]),
247
- validate_foreign_key_code: command_str("validate_foreign_key", [from_table, to_table])
248
- end
249
- end
61
+ check_execute
62
+ when :remove_column, :remove_columns, :remove_timestamps, :remove_reference, :remove_belongs_to
63
+ check_remove_column(method, *args)
64
+ when :remove_index
65
+ check_remove_index(*args)
66
+ when :rename_column
67
+ check_rename_column
68
+ when :rename_table
69
+ check_rename_table
70
+ when :validate_check_constraint
71
+ check_validate_check_constraint
72
+ when :validate_foreign_key
73
+ check_validate_foreign_key
74
+ when :commit_db_transaction
75
+ # if committed, likely no longer in DDL transaction
76
+ # and no longer eligible to be retried at migration level
77
+ # okay to have false positives
78
+ @committed = true
250
79
  end
251
80
 
252
- StrongMigrations.checks.each do |check|
253
- @migration.instance_exec(method, args, &check)
81
+ if !safe?
82
+ # custom checks
83
+ StrongMigrations.checks.each do |check|
84
+ @migration.instance_exec(method, args, &check)
85
+ end
254
86
  end
255
87
  end
256
88
 
257
- result = yield
89
+ result =
90
+ if retry_lock_timeouts?(method)
91
+ # TODO figure out how to handle methods that generate multiple statements
92
+ # like add_reference(table, ref, index: {algorithm: :concurrently})
93
+ # lock timeout after first statement will cause retry to fail
94
+ retry_lock_timeouts { yield }
95
+ else
96
+ yield
97
+ end
258
98
 
259
99
  # outdated statistics + a new index can hurt performance of existing queries
260
100
  if StrongMigrations.auto_analyze && direction == :up && method == :add_index
261
- if postgresql?
262
- # TODO remove verbose in 0.7.0
263
- connection.execute "ANALYZE VERBOSE #{connection.quote_table_name(args[0].to_s)}"
264
- elsif mariadb? || mysql?
265
- connection.execute "ANALYZE TABLE #{connection.quote_table_name(args[0].to_s)}"
266
- end
101
+ adapter.analyze_table(args[0])
267
102
  end
268
103
 
269
104
  result
270
105
  end
271
106
 
272
- def set_timeouts
273
- if !@timeouts_set
274
- if StrongMigrations.statement_timeout
275
- statement =
276
- if postgresql?
277
- "SET statement_timeout TO #{connection.quote(postgresql_timeout(StrongMigrations.statement_timeout))}"
278
- elsif mysql?
279
- "SET max_execution_time = #{connection.quote(StrongMigrations.statement_timeout.to_i * 1000)}"
280
- elsif mariadb?
281
- "SET max_statement_time = #{connection.quote(StrongMigrations.statement_timeout)}"
282
- else
283
- raise StrongMigrations::Error, "Statement timeout not supported for this database"
284
- end
285
-
286
- connection.select_all(statement)
287
- end
288
-
289
- if StrongMigrations.lock_timeout
290
- statement =
291
- if postgresql?
292
- "SET lock_timeout TO #{connection.quote(postgresql_timeout(StrongMigrations.lock_timeout))}"
293
- elsif mysql? || mariadb?
294
- "SET lock_wait_timeout = #{connection.quote(StrongMigrations.lock_timeout)}"
295
- else
296
- raise StrongMigrations::Error, "Lock timeout not supported for this database"
297
- end
298
-
299
- connection.select_all(statement)
107
+ def retry_lock_timeouts(check_committed: false)
108
+ retries = 0
109
+ begin
110
+ yield
111
+ rescue ActiveRecord::LockWaitTimeout => e
112
+ if retries < StrongMigrations.lock_timeout_retries && !(check_committed && @committed)
113
+ retries += 1
114
+ delay = StrongMigrations.lock_timeout_retry_delay
115
+ @migration.say("Lock timeout. Retrying in #{delay} seconds...")
116
+ sleep(delay)
117
+ retry
300
118
  end
301
-
302
- @timeouts_set = true
119
+ raise e
303
120
  end
304
121
  end
305
122
 
306
123
  private
307
124
 
308
- def connection
309
- @migration.connection
310
- end
125
+ def check_version_supported
126
+ return if defined?(@version_checked)
311
127
 
312
- def version
313
- @migration.version
314
- end
315
-
316
- def safe?
317
- @safe || ENV["SAFETY_ASSURED"] || @migration.is_a?(ActiveRecord::Schema) || direction == :down || version_safe?
318
- end
319
-
320
- def version_safe?
321
- version && version <= StrongMigrations.start_after
322
- end
323
-
324
- def postgresql?
325
- connection.adapter_name =~ /postg/i # PostgreSQL, PostGIS
326
- end
327
-
328
- def postgresql_version
329
- @postgresql_version ||= begin
330
- target_version(StrongMigrations.target_postgresql_version) do
331
- # only works with major versions
332
- connection.select_all("SHOW server_version_num").first["server_version_num"].to_i / 10000
128
+ min_version = adapter.min_version
129
+ if min_version
130
+ version = adapter.server_version
131
+ if version < Gem::Version.new(min_version)
132
+ raise UnsupportedVersion, "#{adapter.name} version (#{version}) not supported in this version of Strong Migrations (#{StrongMigrations::VERSION})"
333
133
  end
334
134
  end
335
- end
336
135
 
337
- def mysql?
338
- connection.adapter_name =~ /mysql/i && !connection.try(:mariadb?)
339
- end
340
-
341
- def mysql_version
342
- @mysql_version ||= begin
343
- target_version(StrongMigrations.target_mysql_version) do
344
- connection.select_all("SELECT VERSION()").first["VERSION()"].split("-").first
345
- end
346
- end
136
+ @version_checked = true
347
137
  end
348
138
 
349
- def mariadb?
350
- connection.adapter_name =~ /mysql/i && connection.try(:mariadb?)
351
- end
139
+ def set_timeouts
140
+ return if @timeouts_set
352
141
 
353
- def mariadb_version
354
- @mariadb_version ||= begin
355
- target_version(StrongMigrations.target_mariadb_version) do
356
- connection.select_all("SELECT VERSION()").first["VERSION()"].split("-").first
357
- end
142
+ if StrongMigrations.statement_timeout
143
+ adapter.set_statement_timeout(StrongMigrations.statement_timeout)
144
+ end
145
+ if StrongMigrations.lock_timeout
146
+ adapter.set_lock_timeout(StrongMigrations.lock_timeout)
358
147
  end
359
- end
360
148
 
361
- def target_version(target_version)
362
- version =
363
- if target_version && StrongMigrations.developer_env?
364
- target_version.to_s
365
- else
366
- yield
367
- end
368
- Gem::Version.new(version)
149
+ @timeouts_set = true
369
150
  end
370
151
 
371
152
  def check_lock_timeout
372
- limit = StrongMigrations.lock_timeout_limit
153
+ return if defined?(@lock_timeout_checked)
373
154
 
374
- if limit && !@lock_timeout_checked
375
- if postgresql?
376
- lock_timeout = connection.select_all("SHOW lock_timeout").first["lock_timeout"]
377
- lock_timeout_sec = timeout_to_sec(lock_timeout)
378
- if lock_timeout_sec == 0
379
- warn "[strong_migrations] DANGER: No lock timeout set"
380
- elsif lock_timeout_sec > limit
381
- warn "[strong_migrations] DANGER: Lock timeout is longer than #{limit} seconds: #{lock_timeout}"
382
- end
383
- elsif mysql? || mariadb?
384
- lock_timeout = connection.select_all("SHOW VARIABLES LIKE 'lock_wait_timeout'").first["Value"]
385
- if lock_timeout.to_i > limit
386
- warn "[strong_migrations] DANGER: Lock timeout is longer than #{limit} seconds: #{lock_timeout}"
387
- end
388
- end
389
- @lock_timeout_checked = true
390
- end
391
- end
392
-
393
- # units: https://www.postgresql.org/docs/current/config-setting.html
394
- def timeout_to_sec(timeout)
395
- units = {
396
- "us" => 0.001,
397
- "ms" => 1,
398
- "s" => 1000,
399
- "min" => 1000 * 60,
400
- "h" => 1000 * 60 * 60,
401
- "d" => 1000 * 60 * 60 * 24
402
- }
403
- timeout_ms = timeout.to_i
404
- units.each do |k, v|
405
- if timeout.end_with?(k)
406
- timeout_ms *= v
407
- break
408
- end
155
+ if StrongMigrations.lock_timeout_limit
156
+ adapter.check_lock_timeout(StrongMigrations.lock_timeout_limit)
409
157
  end
410
- timeout_ms / 1000.0
411
- end
412
158
 
413
- def postgresql_timeout(timeout)
414
- if timeout.is_a?(String)
415
- timeout
416
- else
417
- timeout.to_i * 1000
418
- end
159
+ @lock_timeout_checked = true
419
160
  end
420
161
 
421
- def constraints(table_name)
422
- query = <<-SQL
423
- SELECT conname AS name, pg_get_constraintdef(oid) AS def FROM pg_constraint
424
- WHERE contype = 'c'
425
- AND convalidated
426
- AND conrelid = #{connection.quote(connection.quote_table_name(table_name))}::regclass
427
- SQL
428
- connection.select_all(query.squish).to_a
162
+ def safe?
163
+ @safe || ENV["SAFETY_ASSURED"] || (direction == :down && !StrongMigrations.check_down) || version_safe?
429
164
  end
430
165
 
431
- def raise_error(message_key, header: nil, append: nil, **vars)
432
- return unless StrongMigrations.check_enabled?(message_key, version: version)
433
-
434
- message = StrongMigrations.error_messages[message_key] || "Missing message"
435
- message = message + append if append
436
-
437
- vars[:migration_name] = @migration.class.name
438
- vars[:migration_suffix] = "[#{ActiveRecord::VERSION::MAJOR}.#{ActiveRecord::VERSION::MINOR}]"
439
- vars[:base_model] = "ApplicationRecord"
440
-
441
- # escape % not followed by {
442
- message = message.gsub(/%(?!{)/, "%%") % vars if message.include?("%")
443
- @migration.stop!(message, header: header || "Dangerous operation detected")
166
+ def version_safe?
167
+ version && version <= StrongMigrations.start_after
444
168
  end
445
169
 
446
- def constraint_str(statement, identifiers)
447
- # not all identifiers are tables, but this method of quoting should be fine
448
- code = statement % identifiers.map { |v| connection.quote_table_name(v) }
449
- "safety_assured do\n execute '#{code}' \n end"
170
+ def version
171
+ @migration.version
450
172
  end
451
173
 
452
- def command_str(command, args)
453
- str_args = args[0..-2].map { |a| a.inspect }
454
-
455
- # prettier last arg
456
- last_arg = args[-1]
457
- if last_arg.is_a?(Hash)
458
- if last_arg.any?
459
- str_args << last_arg.map do |k, v|
460
- if v.is_a?(Hash)
461
- # pretty index: {algorithm: :concurrently}
462
- "#{k}: {#{v.map { |k2, v2| "#{k2}: #{v2.inspect}" }.join(", ")}}"
174
+ def adapter
175
+ @adapter ||= begin
176
+ cls =
177
+ case connection.adapter_name
178
+ when /postg/i # PostgreSQL, PostGIS
179
+ Adapters::PostgreSQLAdapter
180
+ when /mysql/i
181
+ if connection.try(:mariadb?)
182
+ Adapters::MariaDBAdapter
463
183
  else
464
- "#{k}: #{v.inspect}"
184
+ Adapters::MySQLAdapter
465
185
  end
466
- end.join(", ")
467
- end
468
- else
469
- str_args << last_arg.inspect
470
- end
186
+ else
187
+ Adapters::AbstractAdapter
188
+ end
471
189
 
472
- "#{command} #{str_args.join(", ")}"
190
+ cls.new(self)
191
+ end
473
192
  end
474
193
 
475
- def backfill_code(table, column, default)
476
- model = table.to_s.classify
477
- "#{model}.unscoped.in_batches do |relation| \n relation.update_all #{column}: #{default.inspect}\n sleep(0.01)\n end"
194
+ def connection
195
+ @migration.connection
478
196
  end
479
197
 
480
- def new_table?(table)
481
- @new_tables.include?(table.to_s)
198
+ def retry_lock_timeouts?(method)
199
+ (
200
+ StrongMigrations.lock_timeout_retries > 0 &&
201
+ !in_transaction? &&
202
+ method != :transaction
203
+ )
482
204
  end
483
205
  end
484
206
  end