online_migrations 0.1.0 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 10e5140728aecd56d8f79b52eda79f3244c50212a731db898f3fe36673082bdd
-  data.tar.gz: fa086fff375514e8779ef1d143af68037013373f1954ee4af2596ac3090f1ce9
+  metadata.gz: 3bac7f903ca89cda8d82ee421cae4c1f351a973c88c565f4cecd92ec98b7533c
+  data.tar.gz: e2e2fe22a1d1c70cb753a16fde3ce67890a73f17a058b72fa9383a460837926d
 SHA512:
-  metadata.gz: c18e7f727443671759894c0320c050fbb789ce4015e49eeea751c07f9ec7232e4746c62554493e2118a08b2cc97a519c3645882d3a20c1091ed853a607039d54
-  data.tar.gz: e44ccdb25d5ea1da727e4c6828a55f7426322262388a7a1eb5f022bbd976116550dfbcd04efda289bc7340a5662343d0a283cb4f6fdd756aa7d07313e125dba5
+  metadata.gz: bd297d34d39f7fc1e488f962cb7f11eb0973b29a6b6b3e8330b11db6904430318f85ce60af2fda57adf751294310103c153be5a8511e2fd037477edc17f16dca
+  data.tar.gz: 7fafd129661fd5d63e372a7cb54e55d2715a16a3b4acc6a2d57d5bae69b5f5648723276c244a2ba6918b2330454845901cf34ea393c13e132bb9481ceb7d26a9

data/CHANGELOG.md

@@ -1,5 +1,52 @@
 ## master (unreleased)
 
+## 0.2.0 (2022-01-31)
+
+- Check removing a table with multiple foreign keys
+
+- Check for mismatched reference column types
+
+    For example, it detects cases like:
+
+    ```ruby
+    class AddUserIdToProjects < ActiveRecord::Migration[7.0]
+      def change
+        add_column :projects, :user_id, :integer
+      end
+    end
+    ```
+
+    where `users.id` is of type `bigint`.
+
+- Add support for multiple databases to `start_after` and `target_version` configuration options
+
+    ```ruby
+    OnlineMigrations.configure do |config|
+      config.start_after = { primary: 20211112000000, animals: 20220101000000 }
+      config.target_version = { primary: 10, animals: 14.1 }
+    end
+    ```
+
+- Do not suggest `ignored_columns` when removing columns for Active Record 4.2 (`ignored_columns` was introduced in 5.0)
+
+- Check replacing indexes
+
+    For example, you have an index on `projects.creator_id`. But decide, it is better to have a multicolumn index on `[creator_id, created_at]`:
+
+    ```ruby
+    class AddIndexOnCreationToProjects < ActiveRecord::Migration[7.0]
+      disable_ddl_transaction!
+
+      def change
+        remove_index :projects, :creator_id, algorithm: :concurrently # (1)
+        add_index :projects, [:creator_id, :created_at], algorithm: :concurrently # (2)
+      end
+    end
+    ```
+
+    If there is no existing indexes covering `creator_id`, removing an old index (1) before replacing it with the new one (2) might result in slow queries while building the new index.
+    A safer approach is to swap removing the old and creation of the new index operations.
+
 ## 0.1.0 (2022-01-17)
 
 - First release

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    online_migrations (0.1.0)
+    online_migrations (0.2.0)
       activerecord (>= 4.2)
 
 GEM

data/README.md

@@ -129,18 +129,21 @@ Potentially dangerous operations:
 - [executing SQL directly](#executing-SQL-directly)
 - [adding an index non-concurrently](#adding-an-index-non-concurrently)
 - [removing an index non-concurrently](#removing-an-index-non-concurrently)
+- [replacing an index](#replacing-an-index)
 - [adding a reference](#adding-a-reference)
 - [adding a foreign key](#adding-a-foreign-key)
 - [adding a json column](#adding-a-json-column)
 - [using primary key with short integer type](#using-primary-key-with-short-integer-type)
 - [hash indexes](#hash-indexes)
 - [adding multiple foreign keys](#adding-multiple-foreign-keys)
+- [removing a table with multiple foreign keys](#removing-a-table-with-multiple-foreign-keys)
+- [mismatched reference column types](#mismatched-reference-column-types)
 
 You can also add [custom checks](#custom-checks) or [disable specific checks](#disable-checks).
 
 ### Removing a column
 
-#### Bad
+:x: **Bad**
 
 ActiveRecord caches database columns at runtime, so if you drop a column, it can cause exceptions until your app reboots.
 
@@ -152,14 +155,22 @@ class RemoveNameFromUsers < ActiveRecord::Migration[7.0]
 end
 ```
 
-#### Good
+:white_check_mark: **Good**
 
 1. Ignore the column:
 
   ```ruby
+  # For Active Record 5+
   class User < ApplicationRecord
     self.ignored_columns = ["name"]
   end
+
+  # For Active Record < 5
+  class User < ActiveRecord::Base
+    def self.columns
+      super.reject { |c| c.name == "name" }
+    end
+  end
   ```
 
 2. Deploy
@@ -178,7 +189,7 @@ end
 
 ### Adding a column with a default value
 
-#### Bad
+:x: **Bad**
 
 In earlier versions of PostgreSQL adding a column with a non-null default value to an existing table blocks reads and writes while the entire table is rewritten.
 
@@ -192,7 +203,7 @@ end
 
 In PostgreSQL 11+ this no longer requires a table rewrite and is safe. Volatile expressions, however, such as `random()`, will still result in table rewrites.
 
-#### Good
+:white_check_mark: **Good**
 
 A safer approach is to:
 
@@ -216,7 +227,7 @@ end
 
 ### Backfilling data
 
-#### Bad
+:x: **Bad**
 
 ActiveRecord wraps each migration in a transaction, and backfilling in the same transaction that alters a table keeps the table locked for the [duration of the backfill](https://wework.github.io/data/2015/11/05/add-columns-with-default-values-to-large-tables-in-rails-postgres/).
 
@@ -231,7 +242,7 @@ end
 
 Also, running a single query to update data can cause issues for large tables.
 
-#### Good
+:white_check_mark: **Good**
 
 There are three keys to backfilling safely: batching, throttling, and running it outside a transaction. Use a `update_column_in_batches` helper in a separate migration with `disable_ddl_transaction!`.
 
@@ -256,7 +267,7 @@ end
 
 ### Changing the type of a column
 
-#### Bad
+:x: **Bad**
 
 Changing the type of an existing column blocks reads and writes while the entire table is rewritten.
 
@@ -277,7 +288,7 @@ A few changes don't require a table rewrite (and are safe) in PostgreSQL:
 - Making a `decimal` or `numeric` column unconstrained
 - Changing between `timestamp` and `timestamptz` columns when session time zone is UTC in PostgreSQL 12+
 
-#### Good
+:white_check_mark: **Good**
 
 **Note**: The following steps can also be used to change the primary key's type (e.g., from `integer` to `bigint`).
 
@@ -340,7 +351,7 @@ A safer approach can be accomplished in several steps:
 
 ### Renaming a column
 
-#### Bad
+:x: **Bad**
 
 Renaming a column that's in use will cause errors in your application.
 
@@ -352,7 +363,7 @@ class RenameUsersNameToFirstName < ActiveRecord::Migration[7.0]
 end
 ```
 
-#### Good
+:white_check_mark: **Good**
 
 The "classic" approach suggests creating a new column and copy data/indexes/etc to it from the old column. This can be costly for very large tables. There is a trick that helps to avoid such heavy operations.
 
@@ -415,7 +426,7 @@ end
 
 ### Renaming a table
 
-#### Bad
+:x: **Bad**
 
 Renaming a table that's in use will cause errors in your application.
 
@@ -427,7 +438,7 @@ class RenameClientsToUsers < ActiveRecord::Migration[7.0]
 end
 ```
 
-#### Good
+:white_check_mark: **Good**
 
 The "classic" approach suggests creating a new table and copy data/indexes/etc to it from the old table. This can be costly for very large tables. There is a trick that helps to avoid such heavy operations.
 
@@ -488,7 +499,7 @@ end
 
 ### Creating a table with the force option
 
-#### Bad
+:x: **Bad**
 
 The `force` option can drop an existing table.
 
@@ -502,7 +513,7 @@ class CreateUsers < ActiveRecord::Migration[7.0]
 end
 ```
 
-#### Good
+:white_check_mark: **Good**
 
 Create tables without the `force` option.
 
@@ -520,7 +531,7 @@ If you intend to drop an existing table, run `drop_table` first.
 
 ### Adding a check constraint
 
-#### Bad
+:x: **Bad**
 
 Adding a check constraint blocks reads and writes while every row is checked.
 
@@ -532,7 +543,7 @@ class AddCheckConstraint < ActiveRecord::Migration[7.0]
 end
 ```
 
-#### Good
+:white_check_mark: **Good**
 
 Add the check constraint without validating existing rows, and then validate them in a separate transaction:
 
@@ -551,7 +562,7 @@ end
 
 ### Setting NOT NULL on an existing column
 
-#### Bad
+:x: **Bad**
 
 Setting `NOT NULL` on an existing column blocks reads and writes while every row is checked.
 
@@ -563,7 +574,7 @@ class ChangeUsersNameNull < ActiveRecord::Migration[7.0]
 end
 ```
 
-#### Good
+:white_check_mark: **Good**
 
 Instead, add a check constraint and validate it in a separate transaction:
 
@@ -606,7 +617,7 @@ end
 
 ### Adding an index non-concurrently
 
-#### Bad
+:x: **Bad**
 
 Adding an index non-concurrently blocks writes.
 
@@ -618,7 +629,7 @@ class AddIndexOnUsersEmail < ActiveRecord::Migration[7.0]
 end
 ```
 
-#### Good
+:white_check_mark: **Good**
 
 Add indexes concurrently.
 
@@ -636,7 +647,7 @@ end
 
 ### Removing an index non-concurrently
 
-#### Bad
+:x: **Bad**
 
 While actual removing of an index is usually fast, removing it non-concurrently tries to obtain an `ACCESS EXCLUSIVE` lock on the table, waiting for all existing queries to complete and blocking all the subsequent queries (even `SELECT`s) on that table until the lock is obtained and index is removed.
 
@@ -648,7 +659,7 @@ class RemoveIndexOnUsersEmail < ActiveRecord::Migration[7.0]
 end
 ```
 
-#### Good
+:white_check_mark: **Good**
 
 Remove indexes concurrently.
 
@@ -664,9 +675,43 @@ end
 
 **Note**: If you forget `disable_ddl_transaction!`, the migration will fail.
 
+### Replacing an index
+
+:x: **Bad**
+
+Removing an old index before replacing it with the new one might result in slow queries while building the new index.
+
+```ruby
+class AddIndexOnCreationToProjects < ActiveRecord::Migration[7.0]
+  disable_ddl_transaction!
+
+  def change
+    remove_index :projects, :creator_id, algorithm: :concurrently
+    add_index :projects, [:creator_id, :created_at], algorithm: :concurrently
+  end
+end
+```
+
+**Note**: If removed index is covered by any existing index, then it is safe to remove the index before replacing it with the new one.
+
+:white_check_mark: **Good**
+
+A safer approach is to create the new index and then delete the old one.
+
+```ruby
+class AddIndexOnCreationToProjects < ActiveRecord::Migration[7.0]
+  disable_ddl_transaction!
+
+  def change
+    add_index :projects, [:creator_id, :created_at], algorithm: :concurrently
+    remove_index :projects, :creator_id, algorithm: :concurrently
+  end
+end
+```
+
 ### Adding a reference
 
-#### Bad
+:x: **Bad**
 
 Rails adds an index non-concurrently to references by default, which blocks writes. Additionally, if `foreign_key` option (without `validate: false`) is provided, both tables are blocked while it is validated.
 
@@ -678,7 +723,7 @@ class AddUserToProjects < ActiveRecord::Migration[7.0]
 end
 ```
 
-#### Good
+:white_check_mark: **Good**
 
 Make sure the index is added concurrently and the foreign key is added in a separate migration.
 Or you can use `add_reference_concurrently` helper. It will create a reference and take care of safely adding index and/or foreign key.
@@ -697,7 +742,7 @@ end
 
 ### Adding a foreign key
 
-#### Bad
+:x: **Bad**
 
 Adding a foreign key blocks writes on both tables.
 
@@ -719,7 +764,7 @@ class AddReferenceToProjectsUser < ActiveRecord::Migration[7.0]
 end
 ```
 
-#### Good
+:white_check_mark: **Good**
 
 Add the foreign key without validating existing rows, and then validate them in a separate transaction.
 
@@ -738,7 +783,7 @@ end
 
 ### Adding a json column
 
-#### Bad
+:x: **Bad**
 
 There's no equality operator for the `json` column type, which can cause errors for existing `SELECT DISTINCT` queries in your application.
 
@@ -750,7 +795,7 @@ class AddSettingsToProjects < ActiveRecord::Migration[7.0]
 end
 ```
 
-#### Good
+:white_check_mark: **Good**
 
 Use `jsonb` instead.
 
@@ -764,7 +809,7 @@ end
 
 ### Using primary key with short integer type
 
-#### Bad
+:x: **Bad**
 
 When using short integer types as primary key types, [there is a risk](https://m.signalvnoise.com/update-on-basecamp-3-being-stuck-in-read-only-as-of-nov-8-922am-cst/) of running out of IDs on inserts. The default type in ActiveRecord < 5.1 for primary and foreign keys is `INTEGER`, which allows a little over of 2 billion records. Active Record 5.1 changed the default type to `BIGINT`.
 
@@ -778,7 +823,7 @@ class CreateUsers < ActiveRecord::Migration[7.0]
 end
 ```
 
-#### Good
+:white_check_mark: **Good**
 
 Use one of `bigint`, `bigserial`, `uuid` instead.
 
@@ -794,7 +839,7 @@ end
 
 ### Hash indexes
 
-#### Bad - PostgreSQL < 10
+:x: **Bad - PostgreSQL < 10**
 
 Hash index operations are not WAL-logged, so hash indexes might need to be rebuilt with `REINDEX` after a database crash if there were unwritten changes. Also, changes to hash indexes are not replicated over streaming or file-based replication after the initial base backup, so they give wrong answers to queries that subsequently use them. For these reasons, hash index use is discouraged.
 
@@ -806,7 +851,7 @@ class AddIndexToUsersOnEmail < ActiveRecord::Migration[7.0]
 end
 ```
 
-#### Good - PostgreSQL < 10
+:white_check_mark: **Good - PostgreSQL < 10**
 
 Use B-tree indexes instead.
 
@@ -820,9 +865,9 @@ end
 
 ### Adding multiple foreign keys
 
-#### Bad
+:x: **Bad**
 
-Adding multiple foreign keys in a single migration blocks writes on all involved tables until migration is completed.
+Adding multiple foreign keys in a single migration blocks reads and writes on all involved tables until migration is completed.
 Avoid adding foreign key more than once per migration file, unless the source and target tables are identical.
 
 ```ruby
@@ -836,7 +881,7 @@ class CreateUserProjects < ActiveRecord::Migration[7.0]
 end
 ```
 
-#### Good
+:white_check_mark: **Good**
 
 Add additional foreign keys in separate migration files. See [adding a foreign key](#adding-a-foreign-key) for how to properly add foreign keys.
 
@@ -857,6 +902,82 @@ class AddForeignKeyFromUserProjectsToProject < ActiveRecord::Migration[7.0]
 end
 ```
 
+### Removing a table with multiple foreign keys
+
+:x: **Bad**
+
+Removing a table with multiple foreign keys blocks reads and writes on all involved tables until migration is completed.
+Remove all the foreign keys first.
+
+Assuming, `projects` has foreign keys on `users.id` and `repositories.id`:
+
+```ruby
+class DropProjects < ActiveRecord::Migration[7.0]
+  def change
+    drop_table :projects
+  end
+end
+```
+
+:white_check_mark: **Good**
+
+Remove all the foreign keys first:
+
+```ruby
+class RemoveProjectsUserFk < ActiveRecord::Migration[7.0]
+  def change
+    remove_foreign_key :projects, :users
+  end
+end
+
+class RemoveProjectsRepositoryFk < ActiveRecord::Migration[7.0]
+  def change
+    remove_foreign_key :projects, :repositories
+  end
+end
+```
+
+Then remove the table:
+
+```ruby
+class DropProjects < ActiveRecord::Migration[7.0]
+  def change
+    drop_table :projects
+  end
+end
+```
+
+### Mismatched reference column types
+
+:x: **Bad**
+
+Reference columns should be of the same type as the referenced primary key.
+Otherwise, there's a risk of bugs caused by IDs representable by one type but not the other.
+
+Assuming, there is a `users` table with `bigint` primary key type:
+
+```ruby
+class AddUserIdToProjects < ActiveRecord::Migration[7.0]
+  def change
+    add_column :projects, :user_id, :integer
+  end
+end
+```
+
+:white_check_mark: **Good**
+
+Add a reference column of the same type as a referenced primary key.
+
+Assuming, there is a `users` table with `bigint` primary key type:
+
+```ruby
+class AddUserIdToProjects < ActiveRecord::Migration[7.0]
+  def change
+    add_column :projects, :user_id, :bigint
+  end
+end
+```
+
 ## Assuring Safety
 
 To mark a step in the migration as safe, despite using a method that might otherwise be dangerous, wrap it in a `safety_assured` block.
@@ -986,6 +1107,9 @@ To mark migrations as safe that were created before installing this gem, configu
 # config/initializers/online_migrations.rb
 
 config.start_after = 20220101000000
+
+# or if you use multiple databases (ActiveRecord 6+)
+config.start_after = { primary: 20211112000000, animals: 20220101000000 }
 ```
 
 Use the version from your latest migration.
@@ -998,6 +1122,9 @@ If your development database version is different from production, you can speci
 # config/initializers/online_migrations.rb
 
 config.target_version = 10 # or "12.9" etc
+
+# or if you use multiple databases (ActiveRecord 6+)
+config.target_version = { primary: 10, animals: 14.1 }
 ```
 
 For safety, this option only affects development and test environments. In other environments, the actual server version is always used.

data/lib/online_migrations/background_migrations/advisory_lock.rb

@@ -37,7 +37,7 @@ module OnlineMigrations
         objid = lock_key & 0xffffffff
         classid = (lock_key & (0xffffffff << 32)) >> 32
 
-        active = connection.select_value(<<~SQL)
+        active = connection.select_value(<<-SQL.strip_heredoc)
           SELECT granted
           FROM pg_locks
           WHERE locktype = 'advisory'

data/lib/online_migrations/background_migrations/backfill_column.rb

@@ -13,7 +13,9 @@ module OnlineMigrations
       end
 
       def relation
-        if updates.size == 1 && (column, value = updates.first) && !value.nil?
+        column, value = updates.first
+
+        if updates.size == 1 && !value.nil?
           # If value is nil, the generated SQL is correct (`WHERE column IS NOT NULL`).
           # Otherwise, the SQL is `WHERE column != value`. This condition ignores column
           # with NULLs in it, so we need to also manually check for NULLs.

data/lib/online_migrations/background_migrations/migration_job.rb

@@ -26,7 +26,7 @@ module OnlineMigrations
         stuck_sql             = connection.unprepared_statement { stuck.to_sql }
         failed_retriable_sql  = connection.unprepared_statement { failed_retriable.to_sql }
 
-        from(Arel.sql(<<~SQL))
+        from(Arel.sql(<<-SQL.strip_heredoc))
           (
             (#{failed_retriable_sql})
             UNION

data/lib/online_migrations/background_migrations/migration_job_status_validator.rb

@@ -11,8 +11,9 @@ module OnlineMigrations
       }
 
       def validate(record)
-        return unless (previous_status, new_status = record.status_change)
+        return unless record.status_changed?
 
+        previous_status, new_status = record.status_change
         valid_new_statuses = VALID_STATUS_TRANSITIONS.fetch(previous_status, [])
 
         unless valid_new_statuses.include?(new_status)

data/lib/online_migrations/background_migrations/migration_runner.rb

@@ -79,7 +79,9 @@ module OnlineMigrations
         end
 
         def find_or_create_next_migration_job
-          if (min_value, max_value = migration.next_batch_range)
+          min_value, max_value = migration.next_batch_range
+
+          if min_value && max_value
             create_migration_job!(min_value, max_value)
           else
             migration.migration_jobs.retriable.first

data/lib/online_migrations/background_migrations/migration_status_validator.rb

@@ -29,8 +29,9 @@ module OnlineMigrations
       }
 
       def validate(record)
-        return unless (previous_status, new_status = record.status_change)
+        return unless record.status_changed?
 
+        previous_status, new_status = record.status_change
         valid_new_statuses = VALID_STATUS_TRANSITIONS.fetch(previous_status, [])
 
         unless valid_new_statuses.include?(new_status)

data/lib/online_migrations/change_column_type_helpers.rb

@@ -249,7 +249,7 @@ module OnlineMigrations
             # through PG internal tables.
             # In-depth analysis of implications of this was made, so this approach
             # is considered safe - https://habr.com/ru/company/haulmont/blog/493954/  (in russian).
-            execute(<<~SQL)
+            execute(<<-SQL.strip_heredoc)
               UPDATE pg_catalog.pg_attribute
               SET attnotnull = true
               WHERE attrelid = #{quote(table_name)}::regclass
@@ -471,7 +471,7 @@ module OnlineMigrations
       def __check_constraints(table_name)
         schema = __schema_for_table(table_name)
 
-        check_sql = <<~SQL
+        check_sql = <<-SQL.strip_heredoc
           SELECT
             ccu.column_name as column_name,
             con.conname as constraint_name,
@@ -494,7 +494,7 @@ module OnlineMigrations
       end
 
       def __rename_constraint(table_name, old_name, new_name)
-        execute(<<~SQL)
+        execute(<<-SQL.strip_heredoc)
           ALTER TABLE #{quote_table_name(table_name)}
           RENAME CONSTRAINT #{quote_column_name(old_name)} TO #{quote_column_name(new_name)}
         SQL
@@ -566,7 +566,7 @@ module OnlineMigrations
       def __referencing_table_names(table_name)
         schema = __schema_for_table(table_name)
 
-        select_values(<<~SQL)
+        select_values(<<-SQL.strip_heredoc)
           SELECT DISTINCT con.conrelid::regclass::text AS conrelname
           FROM pg_catalog.pg_constraint con
             INNER JOIN pg_catalog.pg_namespace nsp

data/lib/online_migrations/command_checker.rb

@@ -15,6 +15,7 @@ module OnlineMigrations
       @new_tables = []
       @lock_timeout_checked = false
       @foreign_key_tables = Set.new
+      @removed_indexes = []
     end
 
     def safety_assured
@@ -122,16 +123,23 @@ module OnlineMigrations
       end
 
       def create_join_table(table1, table2, **options, &block)
-        raise_error :create_table if options[:force]
-        raise_error :short_primary_key_type if short_primary_key_type?(options)
+        table_name = options[:table_name] || derive_join_table_name(table1, table2)
+        create_table(table_name, **options, &block)
+      end
 
-        if block
-          collect_foreign_keys(&block)
-          check_for_hash_indexes(&block) if postgresql_version < Gem::Version.new("10")
+      def drop_table(table_name, **_options)
+        foreign_keys = connection.foreign_keys(table_name)
+        referenced_tables = foreign_keys.map(&:to_table).uniq
+        referenced_tables.delete(table_name.to_s) # ignore self references
+
+        if referenced_tables.size > 1
+          raise_error :drop_table_multiple_foreign_keys
         end
+      end
 
+      def drop_join_table(table1, table2, **options)
         table_name = options[:table_name] || derive_join_table_name(table1, table2)
-        @new_tables << table_name.to_s
+        drop_table(table_name, **options)
       end
 
       def change_table(*)
@@ -157,10 +165,13 @@ module OnlineMigrations
             volatile_default: volatile_default
         end
 
-        if type.to_s == "json"
+        if type == :json
           raise_error :add_column_json,
             code: command_str(:add_column, table_name, column_name, :jsonb, options)
         end
+
+        type = :bigint if type == :integer && options[:limit] == 8
+        check_mismatched_foreign_key_type(table_name, column_name, type)
       end
 
       def rename_column(table_name, column_name, new_column, **)
@@ -266,7 +277,7 @@ module OnlineMigrations
 
             if postgresql_version >= Gem::Version.new("12")
               vars[:remove_constraint_code] = command_str(:remove_check_constraint, table_name, name: constraint_name)
-              vars[:change_column_null_code] = command_str(:change_column_null, table_name, column_name, true)
+              vars[:change_column_null_code] = command_str(:change_column_null, table_name, column_name, false)
             end
 
             raise_error :change_column_null, **vars
@@ -283,16 +294,18 @@ module OnlineMigrations
           table_name, *columns = args
         when :remove_timestamps
           table_name = args[0]
-          columns = [:created_at, :updated_at]
+          columns = ["created_at", "updated_at"]
         else
           table_name, reference = args
-          columns = [:"#{reference}_id"]
-          columns << :"#{reference}_type" if options[:polymorphic]
+          columns = ["#{reference}_id"]
+          columns << "#{reference}_type" if options[:polymorphic]
         end
 
+        columns = columns.map(&:to_s)
+
         if !new_table?(table_name)
           indexes = connection.indexes(table_name).select do |index|
-            (index.columns & columns.map(&:to_s)).any?
+            (index.columns & columns).any?
           end
 
           raise_error :remove_column,
@@ -345,15 +358,40 @@ module OnlineMigrations
             bad_index: bad_index,
             bad_foreign_key: bad_foreign_key
         end
+
+        unless options[:polymorphic]
+          type = options[:type] || (Utils.ar_version >= 5.1 ? :bigint : :integer)
+          column_name = "#{ref_name}_id"
+
+          foreign_key_options = foreign_key.is_a?(Hash) ? foreign_key : {}
+          check_mismatched_foreign_key_type(table_name, column_name, type, **foreign_key_options)
+        end
       end
       alias add_belongs_to add_reference
 
       def add_index(table_name, column_name, **options)
         if options[:using].to_s == "hash" && postgresql_version < Gem::Version.new("10")
           raise_error :add_hash_index
-        elsif options[:algorithm] != :concurrently && !new_or_small_table?(table_name)
-          raise_error :add_index,
-            command: command_str(:add_index, table_name, column_name, **options.merge(algorithm: :concurrently))
+        end
+
+        if !new_or_small_table?(table_name)
+          if options[:algorithm] != :concurrently
+            raise_error :add_index,
+              command: command_str(:add_index, table_name, column_name, **options.merge(algorithm: :concurrently))
+          end
+
+          if @removed_indexes.any?
+            index = IndexDefinition.new(table: table_name, columns: column_name, **options)
+            existing_indexes = connection.indexes(table_name)
+
+            @removed_indexes.each do |removed_index|
+              next unless removed_index.intersect?(index)
+
+              unless existing_indexes.any? { |existing_index| removed_index.covered_by?(existing_index) }
+                raise_error :replace_index
+              end
+            end
+          end
         end
       end
 
@@ -364,6 +402,11 @@ module OnlineMigrations
           raise_error :remove_index,
             command: command_str(:remove_index, table_name, **options.merge(algorithm: :concurrently))
         end
+
+        if options[:column] || options[:name]
+          options[:column] ||= connection.indexes(table_name).find { |index| index.name == options[:name].to_s }
+          @removed_indexes << IndexDefinition.new(table: table_name, columns: options.delete(:column), **options)
+        end
       end
 
       def add_foreign_key(from_table, to_table, **options)
@@ -433,7 +476,7 @@ module OnlineMigrations
             nil
           when Hash
             options[:id][:type]
-          when nil
+          when :primary_key, nil
             # default type is used
             connection.native_database_types[:primary_key].split.first
           else
@@ -504,6 +547,7 @@ module OnlineMigrations
         vars[:migration_name] = @migration.name
         vars[:migration_parent] = Utils.migration_parent_string
         vars[:model_parent] = Utils.model_parent_string
+        vars[:ar_version] = Utils.ar_version
 
         if RUBY_VERSION >= "2.6"
           message = ERB.new(template, trim_mode: "<>").result_with_hash(vars)
@@ -545,7 +589,7 @@ module OnlineMigrations
       end
 
       def crud_blocked?
-        locks_query = <<~SQL
+        locks_query = <<-SQL.strip_heredoc
           SELECT relation::regclass::text
           FROM pg_locks
           WHERE mode IN ('ShareLock', 'ShareRowExclusiveLock', 'ExclusiveLock', 'AccessExclusiveLock')
@@ -563,7 +607,7 @@ module OnlineMigrations
       end
 
       def check_constraints(table_name)
-        constraints_query = <<~SQL
+        constraints_query = <<-SQL.strip_heredoc
           SELECT pg_get_constraintdef(oid) AS def
           FROM pg_constraint
           WHERE contype = 'c'
@@ -574,6 +618,34 @@ module OnlineMigrations
         connection.select_all(constraints_query).to_a
       end
 
+      def check_mismatched_foreign_key_type(table_name, column_name, type, **options)
+        column_name = column_name.to_s
+        ref_name = column_name.sub(/_id\z/, "")
+
+        if like_foreign_key?(column_name, type)
+          foreign_table_name = Utils.foreign_table_name(ref_name, options)
+
+          if connection.table_exists?(foreign_table_name)
+            primary_key = options[:primary_key] || connection.primary_key(foreign_table_name)
+            primary_key_column = column_for(foreign_table_name, primary_key)
+
+            if primary_key_column && type != primary_key_column.sql_type.to_sym
+              raise_error :mismatched_foreign_key_type,
+                table_name: table_name, column_name: column_name
+            end
+          end
+        end
+      end
+
+      def like_foreign_key?(column_name, type)
+        column_name.end_with?("_id") &&
+          [:integer, :bigint, :serial, :bigserial, :uuid].include?(type)
+      end
+
+      def column_for(table_name, column_name)
+        connection.columns(table_name).find { |column| column.name == column_name.to_s }
+      end
+
       # From ActiveRecord
       def derive_join_table_name(table1, table2)
         [table1.to_s, table2.to_s].sort.join("\0").gsub(/^(.*_)(.+)\0\1(.+)/, '\1\2_\3').tr("\0", "_")

data/lib/online_migrations/config.rb

@@ -5,20 +5,69 @@ module OnlineMigrations
   class Config
     include ErrorMessages
 
-    # The migration version starting from which checks are performed
+    # Set the migration version starting after which checks are performed
+    # @example
+    #   OnlineMigrations.config.start_after = 20220101000000
+    #
+    # @example Multiple databases
+    #   OnlineMigrations.config.start_after = { primary: 20211112000000, animals: 20220101000000 }
+    #
+    # @note Use the version from your latest migration.
+    #
+    def start_after=(value)
+      if value.is_a?(Hash)
+        ensure_supports_multiple_dbs
+        @start_after = value.stringify_keys
+      else
+        @start_after = value
+      end
+    end
+
+    # The migration version starting after which checks are performed
     # @return [Integer]
     #
-    attr_accessor :start_after
+    def start_after
+      if @start_after.is_a?(Hash)
+        @start_after.fetch(db_config_name) do
+          raise "OnlineMigrations.config.start_after is not configured for :#{db_config_name}"
+        end
+      else
+        @start_after
+      end
+    end
 
-    # The database version against which the checks will be performed
+    # Set the database version against which the checks will be performed
     #
     # If your development database version is different from production, you can specify
     # the production version so the right checks run in development.
     #
-    # @example Set specific target version
+    # @example
     #   OnlineMigrations.config.target_version = 10
     #
-    attr_accessor :target_version
+    # @example Multiple databases
+    #   OnlineMigrations.config.target_version = { primary: 10, animals: 14.1 }
+    #
+    def target_version=(value)
+      if value.is_a?(Hash)
+        ensure_supports_multiple_dbs
+        @target_version = value.stringify_keys
+      else
+        @target_version = value
+      end
+    end
+
+    # The database version against which the checks will be performed
+    # @return [Numeric, String, nil]
+    #
+    def target_version
+      if @target_version.is_a?(Hash)
+        @target_version.fetch(db_config_name) do
+          raise "OnlineMigrations.config.target_version is not configured for :#{db_config_name}"
+        end
+      else
+        @target_version
+      end
+    end
 
     # Whether to perform checks when migrating down
     #
@@ -119,6 +168,7 @@ module OnlineMigrations
 
       @checks = []
       @start_after = 0
+      @target_version = nil
       @small_tables = []
       @check_down = false
       @enabled_checks = @error_messages.keys.map { |k| [k, {}] }.to_h
@@ -194,5 +244,21 @@ module OnlineMigrations
     def add_check(start_after: nil, &block)
       @checks << [{ start_after: start_after }, block]
     end
+
+    private
+      def ensure_supports_multiple_dbs
+        unless Utils.supports_multiple_dbs?
+          raise "Multiple databases are not supported by this ActiveRecord version"
+        end
+      end
+
+      def db_config_name
+        connection = OnlineMigrations.current_migration.connection
+        if Utils.ar_version < 6.1
+          connection.pool.spec.name
+        else
+          connection.pool.db_config.name
+        end
+      end
   end
 end

data/lib/online_migrations/copy_trigger.rb

@@ -23,7 +23,7 @@ module OnlineMigrations
       trigger_name = name(from_columns, to_columns)
       assignment_clauses = assignment_clauses_for_columns(from_columns, to_columns)
 
-      connection.execute(<<~SQL)
+      connection.execute(<<-SQL.strip_heredoc)
         CREATE OR REPLACE FUNCTION #{trigger_name}() RETURNS TRIGGER AS $$
         BEGIN
           #{assignment_clauses};
@@ -32,11 +32,11 @@ module OnlineMigrations
         $$ LANGUAGE plpgsql;
       SQL
 
-      connection.execute(<<~SQL)
+      connection.execute(<<-SQL.strip_heredoc)
         DROP TRIGGER IF EXISTS #{trigger_name} ON #{quoted_table_name}
       SQL
 
-      connection.execute(<<~SQL)
+      connection.execute(<<-SQL.strip_heredoc)
         CREATE TRIGGER #{trigger_name}
           BEFORE INSERT OR UPDATE
           ON #{quoted_table_name}

data/lib/online_migrations/error_messages.rb

@@ -190,12 +190,17 @@ class <%= migration_name %> < <%= migration_parent %>
   def change
     <%= add_constraint_code %>
 <% if backfill_code %>
+
+    # Passing a default value to change_column_null runs a single UPDATE query,
+    # which can cause downtime. Instead, backfill the existing rows in batches.
     <%= backfill_code %>
+
 <% end %>
     <%= validate_constraint_code %>
 <% if remove_constraint_code %>
-    <%= remove_constraint_code %>
+
     <%= change_column_null_code %>
+    <%= remove_constraint_code %>
 <% end %>
   end
 end",
@@ -221,7 +226,13 @@ A safer approach is to:
 1. Ignore the column(s):
 
   class <%= model %> < <%= model_parent %>
+<% if ar_version >= 5 %>
     self.ignored_columns = <%= columns %>
+<% else %>
+    def self.columns
+      super.reject { |c| <%= columns %>.include?(c.name) }
+    end
+<% end %>
   end
 
 2. Deploy
@@ -306,6 +317,10 @@ class <%= migration_name %> < <%= migration_parent %>
   end
 end",
 
+      replace_index:
+"Removing an old index before replacing it with the new one might result in slow queries while building the new index.
+A safer approach is to create the new index and then delete the old one.",
+
       add_foreign_key:
 "Adding a foreign key blocks writes on both tables. Add the foreign key without validating existing rows,
 and then validate them in a separate transaction.
@@ -381,8 +396,16 @@ execute call, so cannot help you here. Make really sure that what
 you're doing is safe before proceeding, then wrap it in a safety_assured { ... } block.",
 
       multiple_foreign_keys:
-"Adding multiple foreign keys in a single migration blocks writes on all involved tables until migration is completed.
+"Adding multiple foreign keys in a single migration blocks reads and writes on all involved tables until migration is completed.
 Avoid adding foreign key more than once per migration file, unless the source and target tables are identical.",
+
+      drop_table_multiple_foreign_keys:
+"Dropping a table with multiple foreign keys blocks reads and writes on all involved tables until migration is completed.
+Remove all the foreign keys first.",
+
+      mismatched_foreign_key_type:
+"<%= table_name %>.<%= column_name %> references a column of different type - foreign keys should be of the same type as the referenced primary key.
+Otherwise, there's a risk of errors caused by IDs representable by one type but not the other.",
     }
   end
 end

data/lib/online_migrations/index_definition.rb

@@ -0,0 +1,52 @@
+# frozen_string_literal: true
+
+module OnlineMigrations
+  # @private
+  class IndexDefinition
+    attr_reader :table, :columns, :unique, :opclasses, :where, :type, :using
+
+    def initialize(**options)
+      @table = options[:table]
+      @columns = Array(options[:columns]).map(&:to_s)
+      @unique = options[:unique]
+      @opclasses = options[:opclass] || {}
+      @where = options[:where]
+      @type = options[:type]
+      @using = options[:using] || :btree
+    end
+
+    # @param other [OnlineMigrations::IndexDefinition, ActiveRecord::ConnectionAdapters::IndexDefinition]
+    def intersect?(other)
+      # For ActiveRecord::ConnectionAdapters::IndexDefinition is for expression indexes,
+      # `columns` is a string
+      table == other.table &&
+        (columns & Array(other.columns)).any?
+    end
+
+    # @param other [OnlineMigrations::IndexDefinition, ActiveRecord::ConnectionAdapters::IndexDefinition]
+    def covered_by?(other)
+      return false if type != other.type
+      return false if using != other.using
+      return false if where != other.where
+      return false if other.respond_to?(:opclasses) && opclasses != other.opclasses
+
+      case [unique, other.unique]
+      when [true, true]
+        columns == other.columns
+      when [true, false]
+        false
+      else
+        prefix?(self, other)
+      end
+    end
+
+    private
+      def prefix?(lhs, rhs)
+        lhs_columns = Array(lhs.columns)
+        rhs_columns = Array(rhs.columns)
+
+        lhs_columns.count <= rhs_columns.count &&
+          rhs_columns[0...lhs_columns.count] == lhs_columns
+      end
+  end
+end

data/lib/online_migrations/indexes_collector.rb

@@ -3,8 +3,6 @@
 module OnlineMigrations
   # @private
   class IndexesCollector
-    IndexDefinition = Struct.new(:using)
-
     COLUMN_TYPES = [:bigint, :binary, :boolean, :date, :datetime, :decimal,
                     :float, :integer, :json, :string, :text, :time, :timestamp, :virtual]
 
@@ -19,7 +17,7 @@ module OnlineMigrations
     end
 
     def index(_column_name, **options)
-      @indexes << IndexDefinition.new(options[:using].to_s)
+      @indexes << IndexDefinition.new(using: options[:using].to_s)
     end
 
     def references(*_ref_names, **options)
@@ -27,7 +25,7 @@ module OnlineMigrations
 
       if index
         using = index.is_a?(Hash) ? index[:using].to_s : nil
-        @indexes << IndexDefinition.new(using)
+        @indexes << IndexDefinition.new(using: using)
       end
     end
     alias belongs_to references
@@ -40,7 +38,7 @@ module OnlineMigrations
 
         if index
           using = index.is_a?(Hash) ? index[:using].to_s : nil
-          @indexes << IndexDefinition.new(using)
+          @indexes << IndexDefinition.new(using: using)
         end
       end
     end

data/lib/online_migrations/schema_statements.rb

@@ -677,7 +677,7 @@ module OnlineMigrations
       index_name = options[:name]
       index_name ||= index_name(table_name, column_names)
 
-      if index_exists?(table_name, column_name, **options)
+      if index_exists?(table_name, column_names, **options)
         disable_statement_timeout do
           # "DROP INDEX CONCURRENTLY" requires a "SHARE UPDATE EXCLUSIVE" lock.
           # It only conflicts with constraint validations, other creating/removing indexes,
@@ -692,7 +692,7 @@ module OnlineMigrations
         end
       else
         Utils.say("Index was not removed because it does not exist (this may be due to an aborted migration "\
-          "or similar): table_name: #{table_name}, column_name: #{column_name}")
+          "or similar): table_name: #{table_name}, column_name: #{column_names}")
       end
     end
 
@@ -701,9 +701,9 @@ module OnlineMigrations
     # @see https://edgeapi.rubyonrails.org/classes/ActiveRecord/ConnectionAdapters/SchemaStatements.html#method-i-add_foreign_key
     #
     def add_foreign_key(from_table, to_table, validate: true, **options)
-      if foreign_key_exists?(from_table, **options.merge(to_table: to_table))
-        message = +"Foreign key was not created because it already exists " \
-          "(this can be due to an aborted migration or similar): from_table: #{from_table}, to_table: #{to_table}"
+      if foreign_key_exists?(from_table, to_table, **options)
+        message = "Foreign key was not created because it already exists " \
+          "(this can be due to an aborted migration or similar): from_table: #{from_table}, to_table: #{to_table}".dup
         message << ", #{options.inspect}" if options.any?
 
         Utils.say(message)
@@ -714,7 +714,7 @@ module OnlineMigrations
         options[:primary_key] ||= "id"
         options[:name] ||= __foreign_key_name(to_table, options[:column])
 
-        query = +<<~SQL
+        query = <<-SQL.strip_heredoc.dup
           ALTER TABLE #{from_table}
           ADD CONSTRAINT #{options[:name]}
           FOREIGN KEY (#{options[:column]})
@@ -853,7 +853,7 @@ module OnlineMigrations
       # ActiveRecord methods
       def __ensure_not_in_transaction!(method_name = caller[0])
         if transaction_open?
-          raise <<~MSG
+          raise <<-MSG.strip_heredoc
             `#{method_name}` cannot run inside a transaction block.
 
             You can remove transaction block by calling `disable_ddl_transaction!` in the body of
@@ -865,7 +865,7 @@ module OnlineMigrations
       def __column_not_nullable?(table_name, column_name)
         schema = __schema_for_table(table_name)
 
-        query = <<~SQL
+        query = <<-SQL.strip_heredoc
           SELECT is_nullable
           FROM information_schema.columns
           WHERE table_schema = #{schema}
@@ -895,7 +895,7 @@ module OnlineMigrations
       end
 
       def __index_column_names(column_names)
-        if column_names.is_a?(String) && /\W/.match?(column_names)
+        if column_names.is_a?(String) && /\W/.match(column_names)
           column_names
         else
           Array(column_names)
@@ -904,7 +904,7 @@ module OnlineMigrations
 
       def __index_valid?(index_name, schema:)
         # ActiveRecord <= 4.2 returns a string, instead of automatically casting to boolean
-        valid = select_value <<~SQL
+        valid = select_value <<-SQL.strip_heredoc
           SELECT indisvalid
           FROM pg_index i
           JOIN pg_class c
@@ -931,7 +931,7 @@ module OnlineMigrations
         when :cascade  then "ON #{action} CASCADE"
         when :restrict then "ON #{action} RESTRICT"
         else
-          raise ArgumentError, <<~MSG
+          raise ArgumentError, <<-MSG.strip_heredoc
             '#{dependency}' is not supported for :on_update or :on_delete.
             Supported values are: :nullify, :cascade, :restrict
           MSG
@@ -981,7 +981,7 @@ module OnlineMigrations
         schema = __schema_for_table(table_name)
         contype = type == :check ? "c" : "f"
 
-        validated = select_value(<<~SQL)
+        validated = select_value(<<-SQL.strip_heredoc)
           SELECT convalidated
           FROM pg_catalog.pg_constraint con
             INNER JOIN pg_catalog.pg_namespace nsp
@@ -1018,7 +1018,7 @@ module OnlineMigrations
       def __check_constraint_exists?(table_name, constraint_name)
         schema = __schema_for_table(table_name)
 
-        check_sql = <<~SQL.squish
+        check_sql = <<-SQL.strip_heredoc
           SELECT COUNT(*)
           FROM pg_catalog.pg_constraint con
             INNER JOIN pg_catalog.pg_class cl

data/lib/online_migrations/utils.rb

@@ -25,6 +25,10 @@ module OnlineMigrations
         Kernel.warn("[online_migrations] #{message}")
       end
 
+      def supports_multiple_dbs?
+        ar_version >= 6.0
+      end
+
       def migration_parent
         if ar_version <= 4.2
           ActiveRecord::Migration
@@ -63,7 +67,7 @@ module OnlineMigrations
       end
 
       def to_bool(value)
-        value.to_s.match?(/^(true|t|yes|y|1|on)$/i)
+        !value.to_s.match(/^(true|t|yes|y|1|on)$/i).nil?
       end
 
       def foreign_table_name(ref_name, options)
@@ -89,7 +93,7 @@ module OnlineMigrations
       def estimated_count(connection, table_name)
         quoted_table = connection.quote(table_name)
 
-        count = connection.select_value(<<~SQL)
+        count = connection.select_value(<<-SQL.strip_heredoc)
           SELECT
             (reltuples / COALESCE(NULLIF(relpages, 0), 1)) *
             (pg_relation_size(#{quoted_table}) / (current_setting('block_size')::integer))
@@ -127,7 +131,7 @@ module OnlineMigrations
       end
 
       def volatile_function?(connection, function_name)
-        query = <<~SQL
+        query = <<-SQL.strip_heredoc
           SELECT provolatile
           FROM pg_catalog.pg_proc
           WHERE proname = #{connection.quote(function_name)}

data/lib/online_migrations/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module OnlineMigrations
-  VERSION = "0.1.0"
+  VERSION = "0.2.0"
 end

data/lib/online_migrations.rb

@@ -11,6 +11,7 @@ require "online_migrations/migrator"
 require "online_migrations/database_tasks"
 require "online_migrations/foreign_key_definition"
 require "online_migrations/foreign_keys_collector"
+require "online_migrations/index_definition"
 require "online_migrations/indexes_collector"
 require "online_migrations/command_checker"
 require "online_migrations/schema_cache"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: online_migrations
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.2.0
 platform: ruby
 authors:
 - fatkodima
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2022-01-16 00:00:00.000000000 Z
+date: 2022-01-31 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activerecord
@@ -80,6 +80,7 @@ files:
 - lib/online_migrations/error_messages.rb
 - lib/online_migrations/foreign_key_definition.rb
 - lib/online_migrations/foreign_keys_collector.rb
+- lib/online_migrations/index_definition.rb
 - lib/online_migrations/indexes_collector.rb
 - lib/online_migrations/lock_retrier.rb
 - lib/online_migrations/migration.rb