strong_migrations 0.6.5 → 0.7.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,28 @@
1
+ require "rails/generators"
2
+
3
+ module StrongMigrations
4
+ module Generators
5
+ class InstallGenerator < Rails::Generators::Base
6
+ source_root File.join(__dir__, "templates")
7
+
8
+ def create_initializer
9
+ template "initializer.rb", "config/initializers/strong_migrations.rb"
10
+ end
11
+
12
+ def start_after
13
+ Time.now.utc.strftime("%Y%m%d%H%M%S")
14
+ end
15
+
16
+ def target_version
17
+ case ActiveRecord::Base.connection_config[:adapter].to_s
18
+ when /mysql/
19
+ # could try to connect to database and check for MariaDB
20
+ # but this should be fine
21
+ '"8.0.12"'
22
+ else
23
+ "10"
24
+ end
25
+ end
26
+ end
27
+ end
28
+ end
@@ -0,0 +1,22 @@
1
+ # Mark existing migrations as safe
2
+ StrongMigrations.start_after = <%= start_after %>
3
+
4
+ # Set timeouts for migrations
5
+ # If you use PgBouncer in transaction mode, delete these lines and set timeouts on the database user
6
+ StrongMigrations.lock_timeout = 10.seconds
7
+ StrongMigrations.statement_timeout = 1.hour
8
+
9
+ # Analyze tables after indexes are added
10
+ # Outdated statistics can sometimes hurt performance
11
+ StrongMigrations.auto_analyze = true
12
+
13
+ # Set the version of the production database
14
+ # so the right checks are run in development
15
+ # StrongMigrations.target_version = <%= target_version %>
16
+
17
+ # Add custom checks
18
+ # StrongMigrations.add_check do |method, args|
19
+ # if method == :add_index && args[0].to_s == "users"
20
+ # stop! "No more indexes on the users table"
21
+ # end
22
+ # end
@@ -5,7 +5,6 @@ require "active_support"
5
5
  require "strong_migrations/checker"
6
6
  require "strong_migrations/database_tasks"
7
7
  require "strong_migrations/migration"
8
- require "strong_migrations/migration_helpers"
9
8
  require "strong_migrations/version"
10
9
 
11
10
  # integrations
@@ -18,14 +17,15 @@ module StrongMigrations
18
17
  class << self
19
18
  attr_accessor :auto_analyze, :start_after, :checks, :error_messages,
20
19
  :target_postgresql_version, :target_mysql_version, :target_mariadb_version,
21
- :enabled_checks, :lock_timeout, :statement_timeout, :helpers
20
+ :enabled_checks, :lock_timeout, :statement_timeout, :check_down, :target_version
21
+ attr_writer :lock_timeout_limit
22
22
  end
23
23
  self.auto_analyze = false
24
24
  self.start_after = 0
25
25
  self.checks = []
26
26
  self.error_messages = {
27
27
  add_column_default:
28
- "Adding a column with a non-null default causes the entire table to be rewritten.
28
+ "Adding a column with a non-null default blocks %{rewrite_blocks} while the entire table is rewritten.
29
29
  Instead, add the column without a default value, then change the default.
30
30
 
31
31
  class %{migration_name} < ActiveRecord::Migration%{migration_suffix}
@@ -50,12 +50,18 @@ class Backfill%{migration_name} < ActiveRecord::Migration%{migration_suffix}
50
50
  end",
51
51
 
52
52
  add_column_json:
53
- "There's no equality operator for the json column type, which can
54
- cause errors for existing SELECT DISTINCT queries. Use jsonb instead.",
53
+ "There's no equality operator for the json column type, which can cause errors for
54
+ existing SELECT DISTINCT queries in your application. Use jsonb instead.
55
+
56
+ class %{migration_name} < ActiveRecord::Migration%{migration_suffix}
57
+ def change
58
+ %{command}
59
+ end
60
+ end",
55
61
 
56
62
  change_column:
57
- "Changing the type of an existing column requires the entire
58
- table and indexes to be rewritten. A safer approach is to:
63
+ "Changing the type of an existing column blocks %{rewrite_blocks}
64
+ while the entire table is rewritten. A safer approach is to:
59
65
 
60
66
  1. Create a new column
61
67
  2. Write to both columns
@@ -64,7 +70,10 @@ table and indexes to be rewritten. A safer approach is to:
64
70
  5. Stop writing to the old column
65
71
  6. Drop the old column",
66
72
 
67
- remove_column: "Active Record caches attributes which causes problems
73
+ change_column_with_not_null:
74
+ "Changing the type is safe, but setting NOT NULL is not.",
75
+
76
+ remove_column: "Active Record caches attributes, which causes problems
68
77
  when removing columns. Be sure to ignore the column%{column_suffix}:
69
78
 
70
79
  class %{model} < %{base_model}
@@ -80,7 +89,8 @@ class %{migration_name} < ActiveRecord::Migration%{migration_suffix}
80
89
  end",
81
90
 
82
91
  rename_column:
83
- "Renaming a column is dangerous. A safer approach is to:
92
+ "Renaming a column that's in use will cause errors
93
+ in your application. A safer approach is to:
84
94
 
85
95
  1. Create a new column
86
96
  2. Write to both columns
@@ -90,7 +100,8 @@ end",
90
100
  6. Drop the old column",
91
101
 
92
102
  rename_table:
93
- "Renaming a table is dangerous. A safer approach is to:
103
+ "Renaming a table that's in use will cause errors
104
+ in your application. A safer approach is to:
94
105
 
95
106
  1. Create a new table. Don't forget to recreate indexes from the old table
96
107
  2. Write to both tables
@@ -111,7 +122,7 @@ class %{migration_name} < ActiveRecord::Migration%{migration_suffix}
111
122
  end",
112
123
 
113
124
  add_index:
114
- "Adding an index non-concurrently locks the table. Instead, use:
125
+ "Adding an index non-concurrently blocks writes. Instead, use:
115
126
 
116
127
  class %{migration_name} < ActiveRecord::Migration%{migration_suffix}
117
128
  disable_ddl_transaction!
@@ -122,7 +133,7 @@ class %{migration_name} < ActiveRecord::Migration%{migration_suffix}
122
133
  end",
123
134
 
124
135
  remove_index:
125
- "Removing an index non-concurrently locks the table. Instead, use:
136
+ "Removing an index non-concurrently blocks writes. Instead, use:
126
137
 
127
138
  class %{migration_name} < ActiveRecord::Migration%{migration_suffix}
128
139
  disable_ddl_transaction!
@@ -165,9 +176,8 @@ class Backfill%{migration_name} < ActiveRecord::Migration%{migration_suffix}
165
176
  end",
166
177
 
167
178
  change_column_null_postgresql:
168
- "Setting NOT NULL on a column requires an AccessExclusiveLock,
169
- which is expensive on large tables. Instead, use a constraint and
170
- validate it in a separate migration with a more agreeable RowShareLock.
179
+ "Setting NOT NULL on an existing column blocks reads and writes while every row is checked.
180
+ Instead, add a check constraint and validate it in a separate migration.
171
181
 
172
182
  class %{migration_name} < ActiveRecord::Migration%{migration_suffix}
173
183
  def change
@@ -181,26 +191,13 @@ class Validate%{migration_name} < ActiveRecord::Migration%{migration_suffix}
181
191
  end
182
192
  end",
183
193
 
184
- change_column_null_postgresql_helper:
185
- "Setting NOT NULL on a column requires an AccessExclusiveLock,
186
- which is expensive on large tables. Instead, we can use a constraint and
187
- validate it in a separate step with a more agreeable RowShareLock.
188
-
189
- class %{migration_name} < ActiveRecord::Migration%{migration_suffix}
190
- disable_ddl_transaction!
191
-
192
- def change
193
- %{command}
194
- end
195
- end",
196
-
197
194
  change_column_null_mysql:
198
195
  "Setting NOT NULL on an existing column is not safe with your database engine.",
199
196
 
200
197
  add_foreign_key:
201
- "New foreign keys are validated by default. This acquires an AccessExclusiveLock,
202
- which is expensive on large tables. Instead, validate it in a separate migration
203
- with a more agreeable RowShareLock.
198
+ "Adding a foreign key blocks writes on both tables. Instead,
199
+ add the foreign key without validating existing rows,
200
+ then validate them in a separate migration.
204
201
 
205
202
  class %{migration_name} < ActiveRecord::Migration%{migration_suffix}
206
203
  def change
@@ -214,21 +211,24 @@ class Validate%{migration_name} < ActiveRecord::Migration%{migration_suffix}
214
211
  end
215
212
  end",
216
213
 
217
- add_foreign_key_helper:
218
- "New foreign keys are validated by default. This acquires an AccessExclusiveLock,
219
- which is expensive on large tables. Instead, we can validate it in a separate step
220
- with a more agreeable RowShareLock.
214
+ validate_foreign_key:
215
+ "Validating a foreign key while writes are blocked is dangerous.
216
+ Use disable_ddl_transaction! or a separate migration."
217
+ }
218
+ self.enabled_checks = (error_messages.keys - [:remove_index]).map { |k| [k, {}] }.to_h
219
+ self.check_down = false
221
220
 
222
- class %{migration_name} < ActiveRecord::Migration%{migration_suffix}
223
- disable_ddl_transaction!
221
+ # private
222
+ def self.developer_env?
223
+ defined?(Rails) && (Rails.env.development? || Rails.env.test?)
224
+ end
224
225
 
225
- def change
226
- %{command}
226
+ def self.lock_timeout_limit
227
+ unless defined?(@lock_timeout_limit)
228
+ @lock_timeout_limit = developer_env? ? false : 10
229
+ end
230
+ @lock_timeout_limit
227
231
  end
228
- end",
229
- }
230
- self.enabled_checks = (error_messages.keys - [:remove_index]).map { |k| [k, {}] }.to_h
231
- self.helpers = false
232
232
 
233
233
  def self.add_check(&block)
234
234
  checks << block
@@ -250,13 +250,6 @@ end",
250
250
  false
251
251
  end
252
252
  end
253
-
254
- # def self.enable_helpers
255
- # unless helpers
256
- # ActiveRecord::Migration.include(StrongMigrations::MigrationHelpers)
257
- # self.helpers = true
258
- # end
259
- # end
260
253
  end
261
254
 
262
255
  ActiveSupport.on_load(:active_record) do
@@ -7,6 +7,7 @@ module StrongMigrations
7
7
  @new_tables = []
8
8
  @safe = false
9
9
  @timeouts_set = false
10
+ @lock_timeout_checked = false
10
11
  end
11
12
 
12
13
  def safety_assured
@@ -21,6 +22,7 @@ module StrongMigrations
21
22
 
22
23
  def perform(method, *args)
23
24
  set_timeouts
25
+ check_lock_timeout
24
26
 
25
27
  unless safe?
26
28
  case method
@@ -94,11 +96,13 @@ Then add the NOT NULL constraint in separate migrations."
94
96
  change_command: command_str("change_column_default", [table, column, default]),
95
97
  remove_command: command_str("remove_column", [table, column]),
96
98
  code: backfill_code(table, column, default),
97
- append: append
99
+ append: append,
100
+ rewrite_blocks: rewrite_blocks
98
101
  end
99
102
 
100
103
  if type.to_s == "json" && postgresql?
101
- raise_error :add_column_json
104
+ raise_error :add_column_json,
105
+ command: command_str("add_column", [table, column, :jsonb, options])
102
106
  end
103
107
  when :change_column
104
108
  table, column, type, options = args
@@ -107,15 +111,24 @@ Then add the NOT NULL constraint in separate migrations."
107
111
  safe = false
108
112
  existing_column = connection.columns(table).find { |c| c.name.to_s == column.to_s }
109
113
  if existing_column
110
- sql_type = existing_column.sql_type.split("(").first
114
+ existing_type = existing_column.sql_type.split("(").first
111
115
  if postgresql?
112
116
  case type.to_s
113
- when "string", "text"
114
- # safe to change limit for varchar
115
- safe = ["character varying", "text"].include?(sql_type)
117
+ when "string"
118
+ # safe to increase limit or remove it
119
+ # not safe to decrease limit or add a limit
120
+ case existing_type
121
+ when "character varying"
122
+ safe = !options[:limit] || (existing_column.limit && options[:limit] >= existing_column.limit)
123
+ when "text"
124
+ safe = !options[:limit]
125
+ end
126
+ when "text"
127
+ # safe to change varchar to text (and text to text)
128
+ safe = ["character varying", "text"].include?(existing_type)
116
129
  when "numeric", "decimal"
117
130
  # numeric and decimal are equivalent and can be used interchangably
118
- safe = ["numeric", "decimal"].include?(sql_type) &&
131
+ safe = ["numeric", "decimal"].include?(existing_type) &&
119
132
  (
120
133
  (
121
134
  # unconstrained
@@ -128,7 +141,7 @@ Then add the NOT NULL constraint in separate migrations."
128
141
  )
129
142
  )
130
143
  when "datetime", "timestamp", "timestamptz"
131
- safe = ["timestamp without time zone", "timestamp with time zone"].include?(sql_type) &&
144
+ safe = ["timestamp without time zone", "timestamp with time zone"].include?(existing_type) &&
132
145
  postgresql_version >= Gem::Version.new("12") &&
133
146
  connection.select_all("SHOW timezone").first["TimeZone"] == "UTC"
134
147
  end
@@ -140,13 +153,19 @@ Then add the NOT NULL constraint in separate migrations."
140
153
  # increased limit, but doesn't change number of length bytes
141
154
  # 1-255 = 1 byte, 256-65532 = 2 bytes, 65533+ = too big for varchar
142
155
  limit = options[:limit] || 255
143
- safe = ["varchar"].include?(sql_type) &&
156
+ safe = ["varchar"].include?(existing_type) &&
144
157
  limit >= existing_column.limit &&
145
158
  (limit <= 255 || existing_column.limit > 255)
146
159
  end
147
160
  end
148
161
  end
149
- raise_error :change_column unless safe
162
+
163
+ # unsafe to set NOT NULL for safe types
164
+ if safe && existing_column.null && options[:null] == false
165
+ raise_error :change_column_with_not_null
166
+ end
167
+
168
+ raise_error :change_column, rewrite_blocks: rewrite_blocks unless safe
150
169
  when :create_table
151
170
  table, options = args
152
171
  options ||= {}
@@ -174,7 +193,7 @@ Then add the NOT NULL constraint in separate migrations."
174
193
  end
175
194
 
176
195
  if options.delete(:foreign_key)
177
- headline = "Adding a validated foreign key locks the table."
196
+ headline = "Adding a foreign key blocks writes on both tables."
178
197
  append = "
179
198
 
180
199
  Then add the foreign key in separate migrations."
@@ -194,16 +213,25 @@ Then add the foreign key in separate migrations."
194
213
  table, column, null, default = args
195
214
  if !null
196
215
  if postgresql?
197
- if helpers?
198
- raise_error :change_column_null_postgresql_helper,
199
- command: command_str(:add_null_constraint_safely, [table, column])
200
- else
216
+ safe = false
217
+ if postgresql_version >= Gem::Version.new("12")
218
+ # TODO likely need to quote the column in some situations
219
+ safe = constraints(table).any? { |c| c["def"] == "CHECK ((#{column} IS NOT NULL))" }
220
+ end
221
+
222
+ unless safe
201
223
  # match https://github.com/nullobject/rein
202
224
  constraint_name = "#{table}_#{column}_null"
203
225
 
226
+ validate_constraint_code = String.new(constraint_str("ALTER TABLE %s VALIDATE CONSTRAINT %s", [table, constraint_name]))
227
+ if postgresql_version >= Gem::Version.new("12")
228
+ validate_constraint_code << "\n #{command_str(:change_column_null, [table, column, null])}"
229
+ validate_constraint_code << "\n #{constraint_str("ALTER TABLE %s DROP CONSTRAINT %s", [table, constraint_name])}"
230
+ end
231
+
204
232
  raise_error :change_column_null_postgresql,
205
233
  add_constraint_code: constraint_str("ALTER TABLE %s ADD CONSTRAINT %s CHECK (%s IS NOT NULL) NOT VALID", [table, constraint_name, column]),
206
- validate_constraint_code: constraint_str("ALTER TABLE %s VALIDATE CONSTRAINT %s", [table, constraint_name])
234
+ validate_constraint_code: validate_constraint_code
207
235
  end
208
236
  elsif mysql? || mariadb?
209
237
  raise_error :change_column_null_mysql
@@ -220,10 +248,7 @@ Then add the foreign key in separate migrations."
220
248
  validate = options.fetch(:validate, true) || ActiveRecord::VERSION::STRING < "5.2"
221
249
 
222
250
  if postgresql? && validate
223
- if helpers?
224
- raise_error :add_foreign_key_helper,
225
- command: command_str(:add_foreign_key_safely, [from_table, to_table, options])
226
- elsif ActiveRecord::VERSION::STRING < "5.2"
251
+ if ActiveRecord::VERSION::STRING < "5.2"
227
252
  # fk name logic from rails
228
253
  primary_key = options[:primary_key] || "id"
229
254
  column = options[:column] || "#{to_table.to_s.singularize}_id"
@@ -239,6 +264,10 @@ Then add the foreign key in separate migrations."
239
264
  validate_foreign_key_code: command_str("validate_foreign_key", [from_table, to_table])
240
265
  end
241
266
  end
267
+ when :validate_foreign_key
268
+ if postgresql? && writes_blocked?
269
+ raise_error :validate_foreign_key
270
+ end
242
271
  end
243
272
 
244
273
  StrongMigrations.checks.each do |check|
@@ -248,9 +277,10 @@ Then add the foreign key in separate migrations."
248
277
 
249
278
  result = yield
250
279
 
280
+ # outdated statistics + a new index can hurt performance of existing queries
251
281
  if StrongMigrations.auto_analyze && direction == :up && method == :add_index
252
282
  if postgresql?
253
- connection.execute "ANALYZE VERBOSE #{connection.quote_table_name(args[0].to_s)}"
283
+ connection.execute "ANALYZE #{connection.quote_table_name(args[0].to_s)}"
254
284
  elsif mariadb? || mysql?
255
285
  connection.execute "ANALYZE TABLE #{connection.quote_table_name(args[0].to_s)}"
256
286
  end
@@ -259,12 +289,14 @@ Then add the foreign key in separate migrations."
259
289
  result
260
290
  end
261
291
 
292
+ private
293
+
262
294
  def set_timeouts
263
295
  if !@timeouts_set
264
296
  if StrongMigrations.statement_timeout
265
297
  statement =
266
298
  if postgresql?
267
- "SET statement_timeout TO #{connection.quote(StrongMigrations.statement_timeout.to_i * 1000)}"
299
+ "SET statement_timeout TO #{connection.quote(postgresql_timeout(StrongMigrations.statement_timeout))}"
268
300
  elsif mysql?
269
301
  "SET max_execution_time = #{connection.quote(StrongMigrations.statement_timeout.to_i * 1000)}"
270
302
  elsif mariadb?
@@ -279,7 +311,7 @@ Then add the foreign key in separate migrations."
279
311
  if StrongMigrations.lock_timeout
280
312
  statement =
281
313
  if postgresql?
282
- "SET lock_timeout TO #{connection.quote(StrongMigrations.lock_timeout.to_i * 1000)}"
314
+ "SET lock_timeout TO #{connection.quote(postgresql_timeout(StrongMigrations.lock_timeout))}"
283
315
  elsif mysql? || mariadb?
284
316
  "SET lock_wait_timeout = #{connection.quote(StrongMigrations.lock_timeout)}"
285
317
  else
@@ -293,8 +325,6 @@ Then add the foreign key in separate migrations."
293
325
  end
294
326
  end
295
327
 
296
- private
297
-
298
328
  def connection
299
329
  @migration.connection
300
330
  end
@@ -304,7 +334,8 @@ Then add the foreign key in separate migrations."
304
334
  end
305
335
 
306
336
  def safe?
307
- @safe || ENV["SAFETY_ASSURED"] || @migration.is_a?(ActiveRecord::Schema) || direction == :down || version_safe?
337
+ @safe || ENV["SAFETY_ASSURED"] || @migration.is_a?(ActiveRecord::Schema) ||
338
+ (direction == :down && !StrongMigrations.check_down) || version_safe?
308
339
  end
309
340
 
310
341
  def version_safe?
@@ -349,8 +380,9 @@ Then add the foreign key in separate migrations."
349
380
  end
350
381
 
351
382
  def target_version(target_version)
383
+ target_version ||= StrongMigrations.target_version
352
384
  version =
353
- if target_version && defined?(Rails) && (Rails.env.development? || Rails.env.test?)
385
+ if target_version && StrongMigrations.developer_env?
354
386
  target_version.to_s
355
387
  else
356
388
  yield
@@ -358,8 +390,69 @@ Then add the foreign key in separate migrations."
358
390
  Gem::Version.new(version)
359
391
  end
360
392
 
361
- def helpers?
362
- StrongMigrations.helpers
393
+ def check_lock_timeout
394
+ limit = StrongMigrations.lock_timeout_limit
395
+
396
+ if limit && !@lock_timeout_checked
397
+ if postgresql?
398
+ lock_timeout = connection.select_all("SHOW lock_timeout").first["lock_timeout"]
399
+ lock_timeout_sec = timeout_to_sec(lock_timeout)
400
+ if lock_timeout_sec == 0
401
+ warn "[strong_migrations] DANGER: No lock timeout set"
402
+ elsif lock_timeout_sec > limit
403
+ warn "[strong_migrations] DANGER: Lock timeout is longer than #{limit} seconds: #{lock_timeout}"
404
+ end
405
+ elsif mysql? || mariadb?
406
+ lock_timeout = connection.select_all("SHOW VARIABLES LIKE 'lock_wait_timeout'").first["Value"]
407
+ if lock_timeout.to_i > limit
408
+ warn "[strong_migrations] DANGER: Lock timeout is longer than #{limit} seconds: #{lock_timeout}"
409
+ end
410
+ end
411
+ @lock_timeout_checked = true
412
+ end
413
+ end
414
+
415
+ # units: https://www.postgresql.org/docs/current/config-setting.html
416
+ def timeout_to_sec(timeout)
417
+ units = {
418
+ "us" => 0.001,
419
+ "ms" => 1,
420
+ "s" => 1000,
421
+ "min" => 1000 * 60,
422
+ "h" => 1000 * 60 * 60,
423
+ "d" => 1000 * 60 * 60 * 24
424
+ }
425
+ timeout_ms = timeout.to_i
426
+ units.each do |k, v|
427
+ if timeout.end_with?(k)
428
+ timeout_ms *= v
429
+ break
430
+ end
431
+ end
432
+ timeout_ms / 1000.0
433
+ end
434
+
435
+ def postgresql_timeout(timeout)
436
+ if timeout.is_a?(String)
437
+ timeout
438
+ else
439
+ timeout.to_i * 1000
440
+ end
441
+ end
442
+
443
+ def constraints(table_name)
444
+ query = <<~SQL
445
+ SELECT
446
+ conname AS name,
447
+ pg_get_constraintdef(oid) AS def
448
+ FROM
449
+ pg_constraint
450
+ WHERE
451
+ contype = 'c' AND
452
+ convalidated AND
453
+ conrelid = #{connection.quote(connection.quote_table_name(table_name))}::regclass
454
+ SQL
455
+ connection.select_all(query.squish).to_a
363
456
  end
364
457
 
365
458
  def raise_error(message_key, header: nil, append: nil, **vars)
@@ -406,6 +499,23 @@ Then add the foreign key in separate migrations."
406
499
  "#{command} #{str_args.join(", ")}"
407
500
  end
408
501
 
502
+ def writes_blocked?
503
+ query = <<~SQL
504
+ SELECT
505
+ relation::regclass::text
506
+ FROM
507
+ pg_locks
508
+ WHERE
509
+ mode IN ('ShareRowExclusiveLock', 'AccessExclusiveLock') AND
510
+ pid = pg_backend_pid()
511
+ SQL
512
+ connection.select_all(query.squish).any?
513
+ end
514
+
515
+ def rewrite_blocks
516
+ mysql? || mariadb? ? "writes" : "reads and writes"
517
+ end
518
+
409
519
  def backfill_code(table, column, default)
410
520
  model = table.to_s.classify
411
521
  "#{model}.unscoped.in_batches do |relation| \n relation.update_all #{column}: #{default.inspect}\n sleep(0.01)\n end"