strong_migrations 0.6.7 → 0.7.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: c2afb67b7b25f7608d3d77d266703a23b5585fcfe006d8442a3da42547979f8d
-  data.tar.gz: 502347f1d82a120f93694bdffb3cc9d434573bb2e6c14f75f61c5dc1b43c723d
+  metadata.gz: '008201c52e7ae0bc2f0614b7b820b2dcb4c99c2ba1d270bd1cd4876bfa7c0258'
+  data.tar.gz: 19907e788d2b789ed04f49cdbaac2db7c6ddfe42315dbe53a440514d4b419019
 SHA512:
-  metadata.gz: bb821b724c1d55150415e117c1509f407edadcb209312ee869b230563b59e55e64072371631b9a75f59e84bbc7af0bdde05e5e5e872ad0dc738917d28ea5c49f
-  data.tar.gz: eb503dd38ecc6ad8877e4a2c506b6503f5a00ac661f8adbc4962d6802009c278091d4a7236dff4c83d64885405d468a9275fd4601e869d3dffac709c3c39cc13
+  metadata.gz: 0231f799c80dc8dedbca766d45cc465d569d872ed2744c2348d31cac279d506e8118aacd5bb478833c7f10c427dc9ceff5c0e8aefe815090dbfbd927aaf0e13c
+  data.tar.gz: ecabbcb9bf64a5e9709f0918497ea31cd36829e06db4a8bec375a87cd63faf6ffc3efc2c33c9ca0f5e524f5d397d01aac41bd31fe36e0af47dd0996c4cb75097

data/CHANGELOG.md

@@ -1,3 +1,29 @@
+## 0.7.3 (2020-11-24)
+
+- Added `safe_by_default` option
+
+## 0.7.2 (2020-10-25)
+
+- Added support for float timeouts
+
+## 0.7.1 (2020-07-27)
+
+- Added `target_version` option to replace database-specific options
+
+## 0.7.0 (2020-07-22)
+
+- Added `check_down` option
+- Added check for `change_column` with `null: false`
+- Added check for `validate_foreign_key`
+- Improved error messages
+- Made auto analyze less verbose in Postgres
+- Decreasing the length limit of a `varchar` column or adding a limit is not safe in Postgres
+- Removed safety checks for `db` rake tasks (Rails 5+ handles this)
+
+## 0.6.8 (2020-05-13)
+
+- `change_column_null` on a column with a `NOT NULL` constraint is safe in Postgres 12+
+
 ## 0.6.7 (2020-05-13)
 
 - Improved comments in initializer

data/LICENSE.txt

@@ -1,4 +1,4 @@
-Copyright (c) 2013 Bob Remeika and David Waller, 2015-2019 Andrew Kane
+Copyright (c) 2013 Bob Remeika and David Waller, 2015-2020 Andrew Kane
 
 MIT License
 

data/README.md

@@ -8,7 +8,7 @@ Supports for PostgreSQL, MySQL, and MariaDB
 
 :tangerine: Battle-tested at [Instacart](https://www.instacart.com/opensource)
 
-[![Build Status](https://travis-ci.org/ankane/strong_migrations.svg?branch=master)](https://travis-ci.org/ankane/strong_migrations)
+[![Build Status](https://github.com/ankane/strong_migrations/workflows/build/badge.svg?branch=master)](https://github.com/ankane/strong_migrations/actions)
 
 ## Installation
 
@@ -25,6 +25,36 @@ bundle install
 rails generate strong_migrations:install
 ```
 
+Strong Migrations sets a long statement timeout for migrations so you can set a [short statement timeout](#app-timeouts) for your application.
+
+## How It Works
+
+When you run a migration that’s potentially dangerous, you’ll see an error message like:
+
+```txt
+=== Dangerous operation detected #strong_migrations ===
+
+Active Record caches attributes, which causes problems
+when removing columns. Be sure to ignore the column:
+
+class User < ApplicationRecord
+  self.ignored_columns = ["name"]
+end
+
+Deploy the code, then wrap this step in a safety_assured { ... } block.
+
+class RemoveColumn < ActiveRecord::Migration[6.0]
+  def change
+    safety_assured { remove_column :users, :name }
+  end
+end
+```
+
+An operation is classified as dangerous if it either:
+
+- Blocks reads or writes for more than a few seconds (after a lock is acquired)
+- Has a good chance of causing application errors
+
 ## Checks
 
 Potentially dangerous operations:
@@ -32,11 +62,11 @@ Potentially dangerous operations:
 - [removing a column](#removing-a-column)
 - [adding a column with a default value](#adding-a-column-with-a-default-value)
 - [backfilling data](#backfilling-data)
-- [changing the type of a column](#renaming-or-changing-the-type-of-a-column)
-- [renaming a column](#renaming-or-changing-the-type-of-a-column)
+- [changing the type of a column](#changing-the-type-of-a-column)
+- [renaming a column](#renaming-a-column)
 - [renaming a table](#renaming-a-table)
 - [creating a table with the force option](#creating-a-table-with-the-force-option)
-- [using change_column_null with a default value](#using-change_column_null-with-a-default-value)
+- [setting NOT NULL on an existing column](#setting-not-null-on-an-existing-column)
 - [executing SQL directly](#executing-SQL-directly)
 
 Postgres-specific checks:
@@ -45,7 +75,6 @@ Postgres-specific checks:
 - [adding a reference](#adding-a-reference)
 - [adding a foreign key](#adding-a-foreign-key)
 - [adding a json column](#adding-a-json-column)
-- [setting NOT NULL on an existing column](#setting-not-null-on-an-existing-column)
 
 Best practices:
 
@@ -57,7 +86,7 @@ You can also add [custom checks](#custom-checks) or [disable specific checks](#d
 
 #### Bad
 
-ActiveRecord caches database columns at runtime, so if you drop a column, it can cause exceptions until your app reboots.
+Active Record caches database columns at runtime, so if you drop a column, it can cause exceptions until your app reboots.
 
 ```ruby
 class RemoveSomeColumnFromUsers < ActiveRecord::Migration[6.0]
@@ -69,7 +98,7 @@ end
 
 #### Good
 
-1. Tell ActiveRecord to ignore the column from its cache
+1. Tell Active Record to ignore the column from its cache
 
   ```ruby
   class User < ApplicationRecord
@@ -92,11 +121,9 @@ end
 
 ### Adding a column with a default value
 
-Note: This operation is safe in Postgres 11+, MySQL 8.0.12+, and MariaDB 10.3.2+.
-
 #### Bad
 
-Adding a column with a default value to an existing table causes the entire table to be rewritten.
+In earlier versions of Postgres, MySQL, and MariaDB, adding a column with a default value to an existing table causes the entire table to be rewritten. During this time, reads and writes are blocked in Postgres, and writes are blocked in MySQL and MariaDB.
 
 ```ruby
 class AddSomeColumnToUsers < ActiveRecord::Migration[6.0]
@@ -106,6 +133,8 @@ class AddSomeColumnToUsers < ActiveRecord::Migration[6.0]
 end
 ```
 
+In Postgres 11+, MySQL 8.0.12+, and MariaDB 10.3.2+, this no longer requires a table rewrite and is safe.
+
 #### Good
 
 Instead, add the column without a default value, then change the default.
@@ -129,7 +158,7 @@ See the next section for how to backfill.
 
 #### Bad
 
-Backfilling in the same transaction that alters a table locks the table for the [duration of the backfill](https://wework.github.io/data/2015/11/05/add-columns-with-default-values-to-large-tables-in-rails-postgres/).
+Active Record creates a transaction around each migration, and backfilling in the same transaction that alters a table keeps the table locked for the [duration of the backfill](https://wework.github.io/data/2015/11/05/add-columns-with-default-values-to-large-tables-in-rails-postgres/).
 
 ```ruby
 class AddSomeColumnToUsers < ActiveRecord::Migration[6.0]
@@ -159,19 +188,11 @@ class BackfillSomeColumn < ActiveRecord::Migration[6.0]
 end
 ```
 
-### Renaming or changing the type of a column
+### Changing the type of a column
 
 #### Bad
 
-```ruby
-class RenameSomeColumn < ActiveRecord::Migration[6.0]
-  def change
-    rename_column :users, :some_column, :new_name
-  end
-end
-```
-
-or
+Changing the type of a column causes the entire table to be rewritten. During this time, reads and writes are blocked in Postgres, and writes are blocked in MySQL and MariaDB.
 
 ```ruby
 class ChangeSomeColumnType < ActiveRecord::Migration[6.0]
@@ -181,17 +202,44 @@ class ChangeSomeColumnType < ActiveRecord::Migration[6.0]
 end
 ```
 
-A few changes are safe in Postgres:
+A few changes don’t require a table rewrite (and are safe) in Postgres:
 
-- Changing between `varchar` and `text` columns
+- Increasing the length limit of a `varchar` column (or removing the limit)
+- Changing a `varchar` column to a `text` column
+- Changing a `text` column to a `varchar` column with no length limit
 - Increasing the precision of a `decimal` or `numeric` column
 - Making a `decimal` or `numeric` column unconstrained
 - Changing between `timestamp` and `timestamptz` columns when session time zone is UTC in Postgres 12+
 
 And a few in MySQL and MariaDB:
 
-- Increasing the length of a `varchar` column from under 255 up to 255
-- Increasing the length of a `varchar` column over 255
+- Increasing the length limit of a `varchar` column from under 255 up to 255
+- Increasing the length limit of a `varchar` column from over 255 to the max limit
+
+#### Good
+
+A safer approach is to:
+
+1. Create a new column
+2. Write to both columns
+3. Backfill data from the old column to the new column
+4. Move reads from the old column to the new column
+5. Stop writing to the old column
+6. Drop the old column
+
+### Renaming a column
+
+#### Bad
+
+Renaming a column that’s in use will cause errors in your application.
+
+```ruby
+class RenameSomeColumn < ActiveRecord::Migration[6.0]
+  def change
+    rename_column :users, :some_column, :new_name
+  end
+end
+```
 
 #### Good
 
@@ -208,6 +256,8 @@ A safer approach is to:
 
 #### Bad
 
+Renaming a table that’s in use will cause errors in your application.
+
 ```ruby
 class RenameUsersToCustomers < ActiveRecord::Migration[6.0]
   def change
@@ -259,33 +309,57 @@ end
 
 If you intend to drop an existing table, run `drop_table` first.
 
-### Using change_column_null with a default value
+### Setting NOT NULL on an existing column
+
+:turtle: Safe by default available
 
 #### Bad
 
-This generates a single `UPDATE` statement to set the default value.
+Setting `NOT NULL` on an existing column blocks reads and writes while every row is checked.
 
 ```ruby
-class ChangeSomeColumnNull < ActiveRecord::Migration[6.0]
+class SetSomeColumnNotNull < ActiveRecord::Migration[6.0]
   def change
-    change_column_null :users, :some_column, false, "default_value"
+    change_column_null :users, :some_column, false
   end
 end
 ```
 
-#### Good
+#### Good - Postgres
 
-Backfill the column [safely](#backfilling-data). Then use:
+Instead, add a check constraint:
 
 ```ruby
-class ChangeSomeColumnNull < ActiveRecord::Migration[6.0]
+class SetSomeColumnNotNull < ActiveRecord::Migration[6.0]
   def change
+    safety_assured do
+      execute 'ALTER TABLE "users" ADD CONSTRAINT "users_some_column_null" CHECK ("some_column" IS NOT NULL) NOT VALID'
+    end
+  end
+end
+```
+
+Then validate it in a separate migration. A `NOT NULL` check constraint is [functionally equivalent](https://medium.com/doctolib/adding-a-not-null-constraint-on-pg-faster-with-minimal-locking-38b2c00c4d1c) to setting `NOT NULL` on the column, but it won’t show up in `schema.rb`. In Postgres 12+, once the check constraint is validated, you can safely set `NOT NULL` on the column and drop the check constraint.
+
+```ruby
+class ValidateSomeColumnNotNull < ActiveRecord::Migration[6.0]
+  def change
+    safety_assured do
+      execute 'ALTER TABLE "users" VALIDATE CONSTRAINT "users_some_column_null"'
+    end
+
+    # in Postgres 12+, you can then safely set NOT NULL on the column
     change_column_null :users, :some_column, false
+    safety_assured do
+      execute 'ALTER TABLE "users" DROP CONSTRAINT "users_some_column_null"'
+    end
   end
 end
 ```
 
-Note: In Postgres, `change_column_null` is still [not safe](#setting-not-null-on-an-existing-column) with this method.
+#### Good - MySQL and MariaDB
+
+[Let us know](https://github.com/ankane/strong_migrations/issues/new) if you have a safe way to do this.
 
 ### Executing SQL directly
 
@@ -301,9 +375,11 @@ end
 
 ### Adding an index non-concurrently
 
+:turtle: Safe by default available
+
 #### Bad
 
-In Postgres, adding an index non-concurrently locks the table.
+In Postgres, adding an index non-concurrently blocks writes.
 
 ```ruby
 class AddSomeIndexToUsers < ActiveRecord::Migration[6.0]
@@ -337,9 +413,11 @@ rails g index table column
 
 ### Adding a reference
 
+:turtle: Safe by default available
+
 #### Bad
 
-Rails adds an index non-concurrently to references by default, which is problematic for Postgres.
+Rails adds an index non-concurrently to references by default, which blocks writes in Postgres.
 
 ```ruby
 class AddReferenceToUsers < ActiveRecord::Migration[6.0]
@@ -365,9 +443,11 @@ end
 
 ### Adding a foreign key
 
+:turtle: Safe by default available
+
 #### Bad
 
-In Postgres, new foreign keys are validated by default, which acquires a `ShareRowExclusiveLock` that can be [expensive on large tables](https://travisofthenorth.com/blog/2017/2/2/postgres-adding-foreign-keys-with-zero-downtime).
+In Postgres, adding a foreign key blocks writes on both tables.
 
 ```ruby
 class AddForeignKeyOnUsers < ActiveRecord::Migration[6.0]
@@ -389,7 +469,7 @@ end
 
 #### Good
 
-Instead, validate it in a separate migration with a more agreeable `RowShareLock`. This approach is documented by Postgres to have “[the least impact on other work](https://www.postgresql.org/docs/current/sql-altertable.html).”
+Add the foreign key without validating existing rows, then validate them in a separate migration.
 
 For Rails 5.2+, use:
 
@@ -401,7 +481,7 @@ class AddForeignKeyOnUsers < ActiveRecord::Migration[6.0]
 end
 ```
 
-Then validate it in a separate migration.
+Then:
 
 ```ruby
 class ValidateForeignKeyOnUsers < ActiveRecord::Migration[6.0]
@@ -423,7 +503,7 @@ class AddForeignKeyOnUsers < ActiveRecord::Migration[5.1]
 end
 ```
 
-Then validate it in a separate migration.
+Then:
 
 ```ruby
 class ValidateForeignKeyOnUsers < ActiveRecord::Migration[5.1]
@@ -439,7 +519,7 @@ end
 
 #### Bad
 
-In Postgres, there’s no equality operator for the `json` column type, which can cause errors for existing `SELECT DISTINCT` queries.
+In Postgres, there’s no equality operator for the `json` column type, which can cause errors for existing `SELECT DISTINCT` queries in your application.
 
 ```ruby
 class AddPropertiesToUsers < ActiveRecord::Migration[6.0]
@@ -461,48 +541,6 @@ class AddPropertiesToUsers < ActiveRecord::Migration[6.0]
 end
 ```
 
-### Setting NOT NULL on an existing column
-
-#### Bad
-
-In Postgres, setting `NOT NULL` on an existing column requires an `AccessExclusiveLock`, which is expensive on large tables.
-
-```ruby
-class SetSomeColumnNotNull < ActiveRecord::Migration[6.0]
-  def change
-    change_column_null :users, :some_column, false
-  end
-end
-```
-
-#### Good
-
-Instead, add a constraint:
-
-```ruby
-class SetSomeColumnNotNull < ActiveRecord::Migration[6.0]
-  def change
-    safety_assured do
-      execute 'ALTER TABLE "users" ADD CONSTRAINT "users_some_column_null" CHECK ("some_column" IS NOT NULL) NOT VALID'
-    end
-  end
-end
-```
-
-Then validate it in a separate migration.
-
-```ruby
-class ValidateSomeColumnNotNull < ActiveRecord::Migration[6.0]
-  def change
-    safety_assured do
-      execute 'ALTER TABLE "users" VALIDATE CONSTRAINT "users_some_column_null"'
-    end
-  end
-end
-```
-
-Note: This is not 100% the same as `NOT NULL` column constraint. Here’s a [good explanation](https://medium.com/doctolib/adding-a-not-null-constraint-on-pg-faster-with-minimal-locking-38b2c00c4d1c).
-
 ### Keeping non-unique indexes to three columns or less
 
 #### Bad
@@ -545,6 +583,20 @@ end
 
 Certain methods like `execute` and `change_table` cannot be inspected and are prevented from running by default. Make sure what you’re doing is really safe and use this pattern.
 
+## Safe by Default [experimental]
+
+Make operations safe by default.
+
+- adding and removing an index
+- adding a foreign key
+- setting NOT NULL on an existing column
+
+Add to `config/initializers/strong_migrations.rb`:
+
+```ruby
+StrongMigrations.safe_by_default = true
+```
+
 ## Custom Checks
 
 Add your own custom checks with:
@@ -581,6 +633,14 @@ StrongMigrations.disable_check(:add_index)
 
 Check the [source code](https://github.com/ankane/strong_migrations/blob/master/lib/strong_migrations.rb) for the list of keys.
 
+## Down Migrations / Rollbacks
+
+By default, checks are disabled when migrating down. Enable them with:
+
+```ruby
+StrongMigrations.check_down = true
+```
+
 ## Custom Messages
 
 To customize specific messages, create an initializer with:
@@ -591,7 +651,7 @@ StrongMigrations.error_messages[:add_column_default] = "Your custom instructions
 
 Check the [source code](https://github.com/ankane/strong_migrations/blob/master/lib/strong_migrations.rb) for the list of keys.
 
-## Timeouts
+## Migration Timeouts
 
 It’s extremely important to set a short lock timeout for migrations. This way, if a migration can’t acquire a lock in a timely manner, other statements won’t be stuck behind it. We also recommend setting a long statement timeout so migrations can run for a while.
 
@@ -611,6 +671,49 @@ ALTER ROLE myuser SET statement_timeout = '1h';
 
 Note: If you use PgBouncer in transaction mode, you must set timeouts on the database user.
 
+## App Timeouts
+
+We recommend adding timeouts to `config/database.yml` to prevent connections from hanging and individual queries from taking up too many resources in controllers, jobs, the Rails console, and other places.
+
+For Postgres:
+
+```yml
+production:
+  connect_timeout: 5
+  variables:
+    statement_timeout: 15s
+    lock_timeout: 10s
+```
+
+Note: If you use PgBouncer in transaction mode, you must set the statement and lock timeouts on the database user as shown above.
+
+For MySQL:
+
+```yml
+production:
+  connect_timeout: 5
+  read_timeout: 5
+  write_timeout: 5
+  variables:
+    max_execution_time: 15000 # ms
+    lock_wait_timeout: 10 # sec
+
+```
+
+For MariaDB:
+
+```yml
+production:
+  connect_timeout: 5
+  read_timeout: 5
+  write_timeout: 5
+  variables:
+    max_statement_time: 15 # sec
+    lock_wait_timeout: 10 # sec
+```
+
+For HTTP connections, Redis, and other services, check out [this guide](https://github.com/ankane/the-ultimate-guide-to-ruby-timeouts).
+
 ## Existing Migrations
 
 To mark migrations as safe that were created before installing this gem, create an initializer with:
@@ -626,11 +729,11 @@ Use the version from your latest migration.
 If your development database version is different from production, you can specify the production version so the right checks run in development.
 
 ```ruby
-StrongMigrations.target_postgresql_version = "10"
-StrongMigrations.target_mysql_version = "8.0.12"
-StrongMigrations.target_mariadb_version = "10.3.2"
+StrongMigrations.target_version = 10 # or "8.0.12", "10.3.2", etc
 ```
 
+The major version works well for Postgres, while the full version is recommended for MySQL and MariaDB.
+
 For safety, this option only affects development and test environments. In other environments, the actual server version is always used.
 
 ## Analyze Tables
@@ -658,22 +761,20 @@ Columns can flip order in `db/schema.rb` when you have multiple developers. One
 task "db:schema:dump": "strong_migrations:alphabetize_columns"
 ```
 
-## Dangerous Tasks
-
-For safety, dangerous database tasks are disabled in production - `db:drop`, `db:reset`, `db:schema:load`, and `db:structure:load`. To get around this, use:
-
-```sh
-SAFETY_ASSURED=1 rails db:drop
-```
-
 ## Permissions
 
 We recommend using a [separate database user](https://ankane.org/postgres-users) for migrations when possible so you don’t need to grant your app user permission to alter tables.
 
+## Smaller Projects
+
+You probably don’t need this gem for smaller projects, as operations that are unsafe at scale can be perfectly safe on smaller, low-traffic tables.
+
 ## Additional Reading
 
 - [Rails Migrations with No Downtime](https://pedro.herokuapp.com/past/2011/7/13/rails_migrations_with_no_downtime/)
 - [PostgreSQL at Scale: Database Schema Changes Without Downtime](https://medium.com/braintree-product-technology/postgresql-at-scale-database-schema-changes-without-downtime-20d3749ed680)
+- [An Overview of DDL Algorithms in MySQL](https://mydbops.wordpress.com/2020/03/04/an-overview-of-ddl-algorithms-in-mysql-covers-mysql-8/)
+- [MariaDB InnoDB Online DDL Overview](https://mariadb.com/kb/en/innodb-online-ddl-overview/)
 
 ## Credits
 

data/lib/generators/strong_migrations/install_generator.rb

@@ -12,6 +12,17 @@ module StrongMigrations
       def start_after
         Time.now.utc.strftime("%Y%m%d%H%M%S")
       end
+
+      def target_version
+        case ActiveRecord::Base.connection_config[:adapter].to_s
+        when /mysql/
+          # could try to connect to database and check for MariaDB
+          # but this should be fine
+          '"8.0.12"'
+        else
+          "10"
+        end
+      end
     end
   end
 end

data/lib/generators/strong_migrations/templates/initializer.rb.tt

@@ -10,6 +10,10 @@ StrongMigrations.statement_timeout = 1.hour
 # Outdated statistics can sometimes hurt performance
 StrongMigrations.auto_analyze = true
 
+# Set the version of the production database
+# so the right checks are run in development
+# StrongMigrations.target_version = <%= target_version %>
+
 # Add custom checks
 # StrongMigrations.add_check do |method, args|
 #   if method == :add_index && args[0].to_s == "users"

data/lib/strong_migrations.rb

@@ -2,6 +2,7 @@
 require "active_support"
 
 # modules
+require "strong_migrations/safe_methods"
 require "strong_migrations/checker"
 require "strong_migrations/database_tasks"
 require "strong_migrations/migration"
@@ -17,15 +18,17 @@ module StrongMigrations
   class << self
     attr_accessor :auto_analyze, :start_after, :checks, :error_messages,
       :target_postgresql_version, :target_mysql_version, :target_mariadb_version,
-      :enabled_checks, :lock_timeout, :statement_timeout
+      :enabled_checks, :lock_timeout, :statement_timeout, :check_down, :target_version,
+      :safe_by_default
     attr_writer :lock_timeout_limit
   end
   self.auto_analyze = false
   self.start_after = 0
   self.checks = []
+  self.safe_by_default = false
   self.error_messages = {
     add_column_default:
-"Adding a column with a non-null default causes the entire table to be rewritten.
+"Adding a column with a non-null default blocks %{rewrite_blocks} while the entire table is rewritten.
 Instead, add the column without a default value, then change the default.
 
 class %{migration_name} < ActiveRecord::Migration%{migration_suffix}
@@ -50,12 +53,18 @@ class Backfill%{migration_name} < ActiveRecord::Migration%{migration_suffix}
 end",
 
     add_column_json:
-"There's no equality operator for the json column type, which can
-cause errors for existing SELECT DISTINCT queries. Use jsonb instead.",
+"There's no equality operator for the json column type, which can cause errors for
+existing SELECT DISTINCT queries in your application. Use jsonb instead.
+
+class %{migration_name} < ActiveRecord::Migration%{migration_suffix}
+  def change
+    %{command}
+  end
+end",
 
     change_column:
-"Changing the type of an existing column requires the entire
-table and indexes to be rewritten. A safer approach is to:
+"Changing the type of an existing column blocks %{rewrite_blocks}
+while the entire table is rewritten. A safer approach is to:
 
 1. Create a new column
 2. Write to both columns
@@ -64,7 +73,10 @@ table and indexes to be rewritten. A safer approach is to:
 5. Stop writing to the old column
 6. Drop the old column",
 
-    remove_column: "Active Record caches attributes which causes problems
+    change_column_with_not_null:
+"Changing the type is safe, but setting NOT NULL is not.",
+
+    remove_column: "Active Record caches attributes, which causes problems
 when removing columns. Be sure to ignore the column%{column_suffix}:
 
 class %{model} < %{base_model}
@@ -80,7 +92,8 @@ class %{migration_name} < ActiveRecord::Migration%{migration_suffix}
 end",
 
     rename_column:
-"Renaming a column is dangerous. A safer approach is to:
+"Renaming a column that's in use will cause errors
+in your application. A safer approach is to:
 
 1. Create a new column
 2. Write to both columns
@@ -90,7 +103,8 @@ end",
 6. Drop the old column",
 
     rename_table:
-"Renaming a table is dangerous. A safer approach is to:
+"Renaming a table that's in use will cause errors
+in your application. A safer approach is to:
 
 1. Create a new table. Don't forget to recreate indexes from the old table
 2. Write to both tables
@@ -111,7 +125,7 @@ class %{migration_name} < ActiveRecord::Migration%{migration_suffix}
 end",
 
     add_index:
-"Adding an index non-concurrently locks the table. Instead, use:
+"Adding an index non-concurrently blocks writes. Instead, use:
 
 class %{migration_name} < ActiveRecord::Migration%{migration_suffix}
   disable_ddl_transaction!
@@ -122,7 +136,7 @@ class %{migration_name} < ActiveRecord::Migration%{migration_suffix}
 end",
 
     remove_index:
-"Removing an index non-concurrently locks the table. Instead, use:
+"Removing an index non-concurrently blocks writes. Instead, use:
 
 class %{migration_name} < ActiveRecord::Migration%{migration_suffix}
   disable_ddl_transaction!
@@ -165,9 +179,8 @@ class Backfill%{migration_name} < ActiveRecord::Migration%{migration_suffix}
 end",
 
     change_column_null_postgresql:
-"Setting NOT NULL on a column requires an AccessExclusiveLock,
-which is expensive on large tables. Instead, use a constraint and
-validate it in a separate migration with a more agreeable RowShareLock.
+"Setting NOT NULL on an existing column blocks reads and writes while every row is checked.
+Instead, add a check constraint and validate it in a separate migration.
 
 class %{migration_name} < ActiveRecord::Migration%{migration_suffix}
   def change
@@ -185,9 +198,9 @@ end",
 "Setting NOT NULL on an existing column is not safe with your database engine.",
 
     add_foreign_key:
-"New foreign keys are validated by default. This acquires an AccessExclusiveLock,
-which is expensive on large tables. Instead, validate it in a separate migration
-with a more agreeable RowShareLock.
+"Adding a foreign key blocks writes on both tables. Instead,
+add the foreign key without validating existing rows,
+then validate them in a separate migration.
 
 class %{migration_name} < ActiveRecord::Migration%{migration_suffix}
   def change
@@ -199,9 +212,14 @@ class Validate%{migration_name} < ActiveRecord::Migration%{migration_suffix}
   def change
     %{validate_foreign_key_code}
   end
-end"
+end",
+
+    validate_foreign_key:
+"Validating a foreign key while writes are blocked is dangerous.
+Use disable_ddl_transaction! or a separate migration."
   }
   self.enabled_checks = (error_messages.keys - [:remove_index]).map { |k| [k, {}] }.to_h
+  self.check_down = false
 
   # private
   def self.developer_env?

data/lib/strong_migrations/checker.rb

@@ -1,6 +1,8 @@
 module StrongMigrations
   class Checker
-    attr_accessor :direction
+    include SafeMethods
+
+    attr_accessor :direction, :transaction_disabled
 
     def initialize(migration)
       @migration = migration
@@ -24,7 +26,7 @@ module StrongMigrations
       set_timeouts
       check_lock_timeout
 
-      unless safe?
+      if !safe? || safe_by_default_method?(method)
         case method
         when :remove_column, :remove_columns, :remove_timestamps, :remove_reference, :remove_belongs_to
           columns =
@@ -65,6 +67,7 @@ module StrongMigrations
             raise_error :add_index_columns, header: "Best practice"
           end
           if postgresql? && options[:algorithm] != :concurrently && !new_table?(table)
+            return safe_add_index(table, columns, options) if StrongMigrations.safe_by_default
             raise_error :add_index, command: command_str("add_index", [table, columns, options.merge(algorithm: :concurrently)])
           end
         when :remove_index
@@ -75,6 +78,7 @@ module StrongMigrations
           options ||= {}
 
           if postgresql? && options[:algorithm] != :concurrently && !new_table?(table)
+            return safe_remove_index(table, options) if StrongMigrations.safe_by_default
             raise_error :remove_index, command: command_str("remove_index", [table, options.merge(algorithm: :concurrently)])
           end
         when :add_column
@@ -96,11 +100,13 @@ Then add the NOT NULL constraint in separate migrations."
               change_command: command_str("change_column_default", [table, column, default]),
               remove_command: command_str("remove_column", [table, column]),
               code: backfill_code(table, column, default),
-              append: append
+              append: append,
+              rewrite_blocks: rewrite_blocks
           end
 
           if type.to_s == "json" && postgresql?
-            raise_error :add_column_json
+            raise_error :add_column_json,
+              command: command_str("add_column", [table, column, :jsonb, options])
           end
         when :change_column
           table, column, type, options = args
@@ -109,15 +115,24 @@ Then add the NOT NULL constraint in separate migrations."
           safe = false
           existing_column = connection.columns(table).find { |c| c.name.to_s == column.to_s }
           if existing_column
-            sql_type = existing_column.sql_type.split("(").first
+            existing_type = existing_column.sql_type.split("(").first
             if postgresql?
               case type.to_s
-              when "string", "text"
-                # safe to change limit for varchar
-                safe = ["character varying", "text"].include?(sql_type)
+              when "string"
+                # safe to increase limit or remove it
+                # not safe to decrease limit or add a limit
+                case existing_type
+                when "character varying"
+                  safe = !options[:limit] || (existing_column.limit && options[:limit] >= existing_column.limit)
+                when "text"
+                  safe = !options[:limit]
+                end
+              when "text"
+                # safe to change varchar to text (and text to text)
+                safe = ["character varying", "text"].include?(existing_type)
               when "numeric", "decimal"
                 # numeric and decimal are equivalent and can be used interchangably
-                safe = ["numeric", "decimal"].include?(sql_type) &&
+                safe = ["numeric", "decimal"].include?(existing_type) &&
                   (
                     (
                       # unconstrained
@@ -130,7 +145,7 @@ Then add the NOT NULL constraint in separate migrations."
                     )
                   )
               when "datetime", "timestamp", "timestamptz"
-                safe = ["timestamp without time zone", "timestamp with time zone"].include?(sql_type) &&
+                safe = ["timestamp without time zone", "timestamp with time zone"].include?(existing_type) &&
                   postgresql_version >= Gem::Version.new("12") &&
                   connection.select_all("SHOW timezone").first["TimeZone"] == "UTC"
               end
@@ -142,13 +157,19 @@ Then add the NOT NULL constraint in separate migrations."
                 # increased limit, but doesn't change number of length bytes
                 # 1-255 = 1 byte, 256-65532 = 2 bytes, 65533+ = too big for varchar
                 limit = options[:limit] || 255
-                safe = ["varchar"].include?(sql_type) &&
+                safe = ["varchar"].include?(existing_type) &&
                   limit >= existing_column.limit &&
                   (limit <= 255 || existing_column.limit > 255)
               end
             end
           end
-          raise_error :change_column unless safe
+
+          # unsafe to set NOT NULL for safe types
+          if safe && existing_column.null && options[:null] == false
+            raise_error :change_column_with_not_null
+          end
+
+          raise_error :change_column, rewrite_blocks: rewrite_blocks unless safe
         when :create_table
           table, options = args
           options ||= {}
@@ -167,16 +188,16 @@ Then add the NOT NULL constraint in separate migrations."
             bad_index = index_value && !concurrently_set
 
             if bad_index || options[:foreign_key]
-              columns = options[:polymorphic] ? [:"#{reference}_type", :"#{reference}_id"] : :"#{reference}_id"
-
               if index_value.is_a?(Hash)
                 options[:index] = options[:index].merge(algorithm: :concurrently)
               else
                 options = options.merge(index: {algorithm: :concurrently})
               end
 
+              return safe_add_reference(table, reference, options) if StrongMigrations.safe_by_default
+
               if options.delete(:foreign_key)
-                headline = "Adding a validated foreign key locks the table."
+                headline = "Adding a foreign key blocks writes on both tables."
                 append = "
 
 Then add the foreign key in separate migrations."
@@ -196,12 +217,34 @@ Then add the foreign key in separate migrations."
           table, column, null, default = args
           if !null
             if postgresql?
-              # match https://github.com/nullobject/rein
-              constraint_name = "#{table}_#{column}_null"
+              safe = false
+              if postgresql_version >= Gem::Version.new("12")
+                # TODO likely need to quote the column in some situations
+                safe = constraints(table).any? { |c| c["def"] == "CHECK ((#{column} IS NOT NULL))" }
+              end
 
-              raise_error :change_column_null_postgresql,
-                add_constraint_code: constraint_str("ALTER TABLE %s ADD CONSTRAINT %s CHECK (%s IS NOT NULL) NOT VALID", [table, constraint_name, column]),
-                validate_constraint_code: constraint_str("ALTER TABLE %s VALIDATE CONSTRAINT %s", [table, constraint_name])
+              unless safe
+                # match https://github.com/nullobject/rein
+                constraint_name = "#{table}_#{column}_null"
+
+                add_code = constraint_str("ALTER TABLE %s ADD CONSTRAINT %s CHECK (%s IS NOT NULL) NOT VALID", [table, constraint_name, column])
+                validate_code = constraint_str("ALTER TABLE %s VALIDATE CONSTRAINT %s", [table, constraint_name])
+                remove_code = constraint_str("ALTER TABLE %s DROP CONSTRAINT %s", [table, constraint_name])
+
+                validate_constraint_code = String.new(safety_assured_str(validate_code))
+                if postgresql_version >= Gem::Version.new("12")
+                  change_args = [table, column, null]
+
+                  validate_constraint_code << "\n    #{command_str(:change_column_null, change_args)}"
+                  validate_constraint_code << "\n    #{safety_assured_str(remove_code)}"
+                end
+
+                return safe_change_column_null(add_code, validate_code, change_args, remove_code) if StrongMigrations.safe_by_default
+
+                raise_error :change_column_null_postgresql,
+                  add_constraint_code: safety_assured_str(add_code),
+                  validate_constraint_code: validate_constraint_code
+              end
             elsif mysql? || mariadb?
               raise_error :change_column_null_mysql
             elsif !default.nil?
@@ -224,15 +267,26 @@ Then add the foreign key in separate migrations."
               hashed_identifier = Digest::SHA256.hexdigest("#{from_table}_#{column}_fk").first(10)
               fk_name = options[:name] || "fk_rails_#{hashed_identifier}"
 
+              add_code = constraint_str("ALTER TABLE %s ADD CONSTRAINT %s FOREIGN KEY (%s) REFERENCES %s (%s) NOT VALID", [from_table, fk_name, column, to_table, primary_key])
+              validate_code = constraint_str("ALTER TABLE %s VALIDATE CONSTRAINT %s", [from_table, fk_name])
+
+              return safe_add_foreign_key_code(from_table, to_table, add_code, validate_code) if StrongMigrations.safe_by_default
+
               raise_error :add_foreign_key,
-                add_foreign_key_code: constraint_str("ALTER TABLE %s ADD CONSTRAINT %s FOREIGN KEY (%s) REFERENCES %s (%s) NOT VALID", [from_table, fk_name, column, to_table, primary_key]),
-                validate_foreign_key_code: constraint_str("ALTER TABLE %s VALIDATE CONSTRAINT %s", [from_table, fk_name])
+                add_foreign_key_code: safety_assured_str(add_code),
+                validate_foreign_key_code: safety_assured_str(validate_code)
             else
+              return safe_add_foreign_key(from_table, to_table, options) if StrongMigrations.safe_by_default
+
               raise_error :add_foreign_key,
                 add_foreign_key_code: command_str("add_foreign_key", [from_table, to_table, options.merge(validate: false)]),
                 validate_foreign_key_code: command_str("validate_foreign_key", [from_table, to_table])
             end
           end
+        when :validate_foreign_key
+          if postgresql? && writes_blocked?
+            raise_error :validate_foreign_key
+          end
         end
 
         StrongMigrations.checks.each do |check|
@@ -245,8 +299,7 @@ Then add the foreign key in separate migrations."
       # outdated statistics + a new index can hurt performance of existing queries
       if StrongMigrations.auto_analyze && direction == :up && method == :add_index
         if postgresql?
-          # TODO remove verbose in 0.7.0
-          connection.execute "ANALYZE VERBOSE #{connection.quote_table_name(args[0].to_s)}"
+          connection.execute "ANALYZE #{connection.quote_table_name(args[0].to_s)}"
         elsif mariadb? || mysql?
           connection.execute "ANALYZE TABLE #{connection.quote_table_name(args[0].to_s)}"
         end
@@ -255,7 +308,8 @@ Then add the foreign key in separate migrations."
       result
     end
 
-    # TODO allow string timeouts in 0.7.0
+    private
+
     def set_timeouts
       if !@timeouts_set
         if StrongMigrations.statement_timeout
@@ -263,7 +317,8 @@ Then add the foreign key in separate migrations."
             if postgresql?
               "SET statement_timeout TO #{connection.quote(postgresql_timeout(StrongMigrations.statement_timeout))}"
             elsif mysql?
-              "SET max_execution_time = #{connection.quote(StrongMigrations.statement_timeout.to_i * 1000)}"
+              # use ceil to prevent no timeout for values under 1 ms
+              "SET max_execution_time = #{connection.quote((StrongMigrations.statement_timeout.to_f * 1000).ceil)}"
             elsif mariadb?
               "SET max_statement_time = #{connection.quote(StrongMigrations.statement_timeout)}"
             else
@@ -290,8 +345,6 @@ Then add the foreign key in separate migrations."
       end
     end
 
-    private
-
     def connection
       @migration.connection
     end
@@ -301,7 +354,8 @@ Then add the foreign key in separate migrations."
     end
 
     def safe?
-      @safe || ENV["SAFETY_ASSURED"] || @migration.is_a?(ActiveRecord::Schema) || direction == :down || version_safe?
+      @safe || ENV["SAFETY_ASSURED"] || @migration.is_a?(ActiveRecord::Schema) ||
+        (direction == :down && !StrongMigrations.check_down) || version_safe?
     end
 
     def version_safe?
@@ -346,6 +400,7 @@ Then add the foreign key in separate migrations."
     end
 
     def target_version(target_version)
+      target_version ||= StrongMigrations.target_version
       version =
         if target_version && StrongMigrations.developer_env?
           target_version.to_s
@@ -369,6 +424,7 @@ Then add the foreign key in separate migrations."
           end
         elsif mysql? || mariadb?
           lock_timeout = connection.select_all("SHOW VARIABLES LIKE 'lock_wait_timeout'").first["Value"]
+          # lock timeout is an integer
           if lock_timeout.to_i > limit
             warn "[strong_migrations] DANGER: Lock timeout is longer than #{limit} seconds: #{lock_timeout}"
           end
@@ -379,7 +435,7 @@ Then add the foreign key in separate migrations."
 
     # units: https://www.postgresql.org/docs/current/config-setting.html
     def timeout_to_sec(timeout)
-      suffixes = {
+      units = {
         "us" => 0.001,
         "ms" => 1,
         "s" => 1000,
@@ -388,7 +444,7 @@ Then add the foreign key in separate migrations."
         "d" => 1000 * 60 * 60 * 24
       }
       timeout_ms = timeout.to_i
-      suffixes.each do |k, v|
+      units.each do |k, v|
         if timeout.end_with?(k)
           timeout_ms *= v
           break
@@ -401,10 +457,26 @@ Then add the foreign key in separate migrations."
       if timeout.is_a?(String)
         timeout
       else
-        timeout.to_i * 1000
+        # use ceil to prevent no timeout for values under 1 ms
+        (timeout.to_f * 1000).ceil
       end
     end
 
+    def constraints(table_name)
+      query = <<~SQL
+        SELECT
+          conname AS name,
+          pg_get_constraintdef(oid) AS def
+        FROM
+          pg_constraint
+        WHERE
+          contype = 'c' AND
+          convalidated AND
+          conrelid = #{connection.quote(connection.quote_table_name(table_name))}::regclass
+      SQL
+      connection.select_all(query.squish).to_a
+    end
+
     def raise_error(message_key, header: nil, append: nil, **vars)
       return unless StrongMigrations.check_enabled?(message_key, version: version)
 
@@ -422,7 +494,10 @@ Then add the foreign key in separate migrations."
 
     def constraint_str(statement, identifiers)
       # not all identifiers are tables, but this method of quoting should be fine
-      code = statement % identifiers.map { |v| connection.quote_table_name(v) }
+      statement % identifiers.map { |v| connection.quote_table_name(v) }
+    end
+
+    def safety_assured_str(code)
       "safety_assured do\n      execute '#{code}' \n    end"
     end
 
@@ -449,6 +524,23 @@ Then add the foreign key in separate migrations."
       "#{command} #{str_args.join(", ")}"
     end
 
+    def writes_blocked?
+      query = <<~SQL
+        SELECT
+          relation::regclass::text
+        FROM
+          pg_locks
+        WHERE
+          mode IN ('ShareRowExclusiveLock', 'AccessExclusiveLock') AND
+          pid = pg_backend_pid()
+      SQL
+      connection.select_all(query.squish).any?
+    end
+
+    def rewrite_blocks
+      mysql? || mariadb? ? "writes" : "reads and writes"
+    end
+
     def backfill_code(table, column, default)
       model = table.to_s.classify
       "#{model}.unscoped.in_batches do |relation| \n      relation.update_all #{column}: #{default.inspect}\n      sleep(0.01)\n    end"

data/lib/strong_migrations/migration.rb

@@ -3,6 +3,7 @@ module StrongMigrations
     def migrate(direction)
       strong_migrations_checker.direction = direction
       super
+      connection.begin_db_transaction if strong_migrations_checker.transaction_disabled
     end
 
     def method_missing(method, *args)

data/lib/strong_migrations/railtie.rb

@@ -5,10 +5,6 @@ module StrongMigrations
   class Railtie < Rails::Railtie
     rake_tasks do
       load "tasks/strong_migrations.rake"
-
-      ["db:drop", "db:reset", "db:schema:load", "db:structure:load"].each do |t|
-        Rake::Task[t].enhance ["strong_migrations:safety_assured"]
-      end
     end
   end
 end

data/lib/strong_migrations/safe_methods.rb

@@ -0,0 +1,112 @@
+module StrongMigrations
+  module SafeMethods
+    def safe_by_default_method?(method)
+      StrongMigrations.safe_by_default && [:add_index, :add_belongs_to, :add_reference, :remove_index, :add_foreign_key, :change_column_null].include?(method)
+    end
+
+    # TODO check if invalid index with expected name exists and remove if needed
+    def safe_add_index(table, columns, options)
+      disable_transaction
+      @migration.add_index(table, columns, options.merge(algorithm: :concurrently))
+    end
+
+    def safe_remove_index(table, options)
+      disable_transaction
+      @migration.remove_index(table, options.merge(algorithm: :concurrently))
+    end
+
+    def safe_add_reference(table, reference, options)
+      @migration.reversible do |dir|
+        dir.up do
+          disable_transaction
+          foreign_key = options.delete(:foreign_key)
+          @migration.add_reference(table, reference, **options)
+          if foreign_key
+            # same as Active Record
+            name =
+              if foreign_key.is_a?(Hash) && foreign_key[:to_table]
+                foreign_key[:to_table]
+              else
+                (ActiveRecord::Base.pluralize_table_names ? reference.to_s.pluralize : reference).to_sym
+              end
+
+            @migration.add_foreign_key(table, name)
+          end
+        end
+        dir.down do
+          @migration.remove_reference(table, reference)
+        end
+      end
+    end
+
+    def safe_add_foreign_key(from_table, to_table, options)
+      @migration.reversible do |dir|
+        dir.up do
+          @migration.add_foreign_key(from_table, to_table, **options.merge(validate: false))
+          disable_transaction
+          @migration.validate_foreign_key(from_table, to_table)
+        end
+        dir.down do
+          @migration.remove_foreign_key(from_table, to_table)
+        end
+      end
+    end
+
+    def safe_add_foreign_key_code(from_table, to_table, add_code, validate_code)
+      @migration.reversible do |dir|
+        dir.up do
+          @migration.safety_assured do
+            @migration.execute(add_code)
+            disable_transaction
+            @migration.execute(validate_code)
+          end
+        end
+        dir.down do
+          @migration.remove_foreign_key(from_table, to_table)
+        end
+      end
+    end
+
+    def safe_change_column_null(add_code, validate_code, change_args, remove_code)
+      @migration.reversible do |dir|
+        dir.up do
+          @migration.safety_assured do
+            @migration.execute(add_code)
+            disable_transaction
+            @migration.execute(validate_code)
+          end
+          if change_args
+            @migration.change_column_null(*change_args)
+            @migration.safety_assured do
+              @migration.execute(remove_code)
+            end
+          end
+        end
+        dir.down do
+          if change_args
+            down_args = change_args.dup
+            down_args[2] = true
+            @migration.change_column_null(*down_args)
+          else
+            @migration.safety_assured do
+              @migration.execute(remove_code)
+            end
+          end
+        end
+      end
+    end
+
+    # hard to commit at right time when reverting
+    # so just commit at start
+    def disable_transaction
+      if in_transaction? && !transaction_disabled
+        @migration.connection.commit_db_transaction
+        self.transaction_disabled = true
+      end
+    end
+
+    def in_transaction?
+      @migration.connection.open_transactions > 0
+    end
+  end
+end

data/lib/strong_migrations/version.rb

@@ -1,3 +1,3 @@
 module StrongMigrations
-  VERSION = "0.6.7"
+  VERSION = "0.7.3"
 end

data/lib/tasks/strong_migrations.rake

@@ -1,10 +1,4 @@
-# https://nithinbekal.com/posts/safe-rake-tasks
-
 namespace :strong_migrations do
-  task safety_assured: :environment do
-    raise "Set SAFETY_ASSURED=1 to run this task in production" if Rails.env.production? && !ENV["SAFETY_ASSURED"]
-  end
-
   # https://www.pgrs.net/2008/03/13/alphabetize-schema-rb-columns/
   task :alphabetize_columns do
     $stderr.puts "Dumping schema"

metadata

@@ -1,16 +1,16 @@
 --- !ruby/object:Gem::Specification
 name: strong_migrations
 version: !ruby/object:Gem::Version
-  version: 0.6.7
+  version: 0.7.3
 platform: ruby
 authors:
 - Andrew Kane
 - Bob Remeika
 - David Waller
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2020-05-13 00:00:00.000000000 Z
+date: 2020-11-24 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activerecord
@@ -96,7 +96,7 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-description: 
+description:
 email:
 - andrew@chartkick.com
 - bob.remeika@gmail.com
@@ -116,13 +116,14 @@ files:
 - lib/strong_migrations/database_tasks.rb
 - lib/strong_migrations/migration.rb
 - lib/strong_migrations/railtie.rb
+- lib/strong_migrations/safe_methods.rb
 - lib/strong_migrations/version.rb
 - lib/tasks/strong_migrations.rake
 homepage: https://github.com/ankane/strong_migrations
 licenses:
 - MIT
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -137,8 +138,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.2
-signing_key: 
+rubygems_version: 3.1.4
+signing_key:
 specification_version: 4
 summary: Catch unsafe migrations in development
 test_files: []