online_migrations 0.1.0

data/lib/online_migrations/command_recorder.rb

@@ -0,0 +1,137 @@
+# frozen_string_literal: true
+
+module OnlineMigrations
+  # @private
+  module CommandRecorder
+    REVERSIBLE_AND_IRREVERSIBLE_METHODS = [
+      :update_column_in_batches,
+      :initialize_column_rename,
+      :revert_initialize_column_rename,
+      :finalize_column_rename,
+      :revert_finalize_column_rename,
+      :initialize_table_rename,
+      :revert_initialize_table_rename,
+      :finalize_table_rename,
+      :revert_finalize_table_rename,
+      :swap_column_names,
+      :add_column_with_default,
+      :add_not_null_constraint,
+      :add_text_limit_constraint,
+      :add_reference_concurrently,
+      :change_column_type_in_background,
+      :enqueue_background_migration,
+
+      # column type change helpers
+      :initialize_column_type_change,
+      :initialize_columns_type_change,
+      :revert_initialize_column_type_change,
+      :revert_initialize_columns_type_change,
+      :backfill_column_for_type_change,
+      :backfill_columns_for_type_change,
+      :finalize_column_type_change,
+      :finalize_columns_type_change,
+      :revert_finalize_column_type_change,
+      :cleanup_column_type_change,
+      :cleanup_columns_type_change,
+    ]
+
+    REVERSIBLE_AND_IRREVERSIBLE_METHODS.each do |method|
+      class_eval <<-RUBY, __FILE__, __LINE__ + 1
+        def #{method}(*args, &block)          # def create_table(*args, &block)
+          record(:"#{method}", args, &block)  #   record(:create_table, args, &block)
+        end                                   # end
+      RUBY
+    end
+
+    private
+      module StraightReversions
+        {
+          initialize_column_rename:           :revert_initialize_column_rename,
+          finalize_column_rename:             :revert_finalize_column_rename,
+          initialize_table_rename:            :revert_initialize_table_rename,
+          finalize_table_rename:              :revert_finalize_table_rename,
+          add_not_null_constraint:            :remove_not_null_constraint,
+          initialize_column_type_change:      :revert_initialize_column_type_change,
+          initialize_columns_type_change:     :revert_initialize_columns_type_change,
+          finalize_column_type_change:        :revert_finalize_column_type_change,
+          finalize_columns_type_change:       :revert_finalize_columns_type_change,
+        }.each do |cmd, inv|
+          [[inv, cmd], [cmd, inv]].each do |method, inverse|
+            class_eval <<-RUBY, __FILE__, __LINE__ + 1
+              def invert_#{method}(args, &block)    # def invert_create_table(args, &block)
+                [:#{inverse}, args, block]          #   [:drop_table, args, block]
+              end                                   # end
+            RUBY
+          end
+        end
+      end
+
+      include StraightReversions
+
+      def invert_swap_column_names(args)
+        table_name, column1, column2 = args
+        [:swap_column_names, [table_name, column2, column1]]
+      end
+
+      def invert_add_column_with_default(args)
+        table_name, column_name, = args
+        [:remove_column, [table_name, column_name]]
+      end
+
+      def invert_revert_initialize_column_rename(args)
+        _table, column, new_column = args
+        if !column || !new_column
+          raise ActiveRecord::IrreversibleMigration,
+            "invert_revert_initialize_column_rename is only reversible if given a column and new_column."
+        end
+        [:initialize_column_rename, args]
+      end
+
+      def invert_finalize_table_rename(args)
+        _table_name, new_name = args
+        unless new_name
+          raise ActiveRecord::IrreversibleMigration,
+            "finalize_table_rename is only reversible if given a new_name."
+        end
+        [:revert_finalize_table_rename, args]
+      end
+
+      def invert_revert_initialize_column_type_change(args)
+        unless args[2]
+          raise ActiveRecord::IrreversibleMigration,
+            "revert_initialize_column_type_change is only reversible if given a new_type."
+        end
+        super
+      end
+
+      def invert_revert_initialize_columns_type_change(args)
+        if args[1].empty?
+          raise ActiveRecord::IrreversibleMigration,
+            "revert_initialize_columns_type_change is only reversible if given a columns_and_types."
+        end
+        super
+      end
+
+      def invert_add_not_null_constraint(args)
+        options = args.extract_options!
+        table_name, column = args
+        options.delete(:validate)
+        [:remove_not_null_constraint, [table_name, column, **options]]
+      end
+
+      def invert_add_text_limit_constraint(args)
+        options = args.extract_options!
+        table_name, column, _limit = args
+        options.delete(:validate)
+        [:remove_text_limit_constraint, [table_name, column, **options]]
+      end
+
+      def invert_remove_text_limit_constraint(args)
+        unless args[2]
+          raise ActiveRecord::IrreversibleMigration, "remove_text_limit_constraint is only reversible if given a limit."
+        end
+
+        super
+      end
+  end
+end

data/lib/online_migrations/config.rb

@@ -0,0 +1,198 @@
+# frozen_string_literal: true
+
+module OnlineMigrations
+  # Class representing configuration options for the gem.
+  class Config
+    include ErrorMessages
+
+    # The migration version starting from which checks are performed
+    # @return [Integer]
+    #
+    attr_accessor :start_after
+
+    # The database version against which the checks will be performed
+    #
+    # If your development database version is different from production, you can specify
+    # the production version so the right checks run in development.
+    #
+    # @example Set specific target version
+    #   OnlineMigrations.config.target_version = 10
+    #
+    attr_accessor :target_version
+
+    # Whether to perform checks when migrating down
+    #
+    # Disabled by default
+    # @return [Boolean]
+    #
+    attr_accessor :check_down
+
+    # Error messages
+    #
+    # @return [Hash] Keys are error names, values are error messages
+    # @example To change a message
+    #   OnlineMigrations.config.error_messages[:remove_column] = "Your custom instructions"
+    #
+    attr_accessor :error_messages
+
+    # Maximum allowed lock timeout value (in seconds)
+    #
+    # If set lock timeout is greater than this value, the migration will fail.
+    # The default value is 10 seconds.
+    #
+    # @return [Numeric]
+    #
+    attr_accessor :lock_timeout_limit
+
+    # List of tables with permanently small number of records
+    #
+    # These are usually tables like "settings", "prices", "plans" etc.
+    # It is considered safe to perform most of the dangerous operations on them,
+    #   like adding indexes, columns etc.
+    #
+    # @return [Array<String, Symbol>]
+    #
+    attr_reader :small_tables
+
+    # Tables that are in the process of being renamed
+    #
+    # @return [Hash] Keys are old table names, values - new table names
+    # @example To add a table
+    #   OnlineMigrations.config.table_renames["users"] = "clients"
+    #
+    attr_accessor :table_renames
+
+    # Columns that are in the process of being renamed
+    #
+    # @return [Hash] Keys are table names, values - hashes with old column names as keys
+    #   and new column names as values
+    # @example To add a column
+    #   OnlineMigrations.config.column_renames["users] = { "name" => "first_name" }
+    #
+    attr_accessor :column_renames
+
+    # Lock retrier in use (see LockRetrier)
+    #
+    # No retries are performed by default.
+    # @return [OnlineMigrations::LockRetrier]
+    #
+    attr_reader :lock_retrier
+
+    # Returns a list of custom checks
+    #
+    # Use `add_check` to add custom checks
+    #
+    # @return [Array<Array<Hash>, Proc>]
+    #
+    attr_reader :checks
+
+    # Returns a list of enabled checks
+    #
+    # All checks are enabled by default. To disable/enable a check use `disable_check`/`enable_check`.
+    # For the list of available checks look at `lib/error_messages` folder.
+    #
+    # @return [Array]
+    #
+    attr_reader :enabled_checks
+
+    # Configuration object to configure background migrations
+    #
+    # @return [BackgroundMigrationsConfig]
+    # @see BackgroundMigrationsConfig
+    #
+    attr_reader :background_migrations
+
+    def initialize
+      @table_renames = {}
+      @column_renames = {}
+      @error_messages = ERROR_MESSAGES
+      @lock_timeout_limit = 10.seconds
+
+      @lock_retrier = ExponentialLockRetrier.new(
+        attempts: 30,
+        base_delay: 0.01.seconds,
+        max_delay: 1.minute,
+        lock_timeout: 0.05.seconds
+      )
+
+      @background_migrations = BackgroundMigrations::Config.new
+
+      @checks = []
+      @start_after = 0
+      @small_tables = []
+      @check_down = false
+      @enabled_checks = @error_messages.keys.map { |k| [k, {}] }.to_h
+    end
+
+    def lock_retrier=(value)
+      @lock_retrier = value || NullLockRetrier.new
+    end
+
+    def small_tables=(table_names)
+      @small_tables = table_names.map(&:to_s)
+    end
+
+    # Enables specific check
+    #
+    # For the list of available checks look at `lib/error_messages` module.
+    #
+    # @param name [Symbol] check name
+    # @param start_after [Integer] migration version from which this check will be performed
+    # @return [void]
+    #
+    def enable_check(name, start_after: nil)
+      enabled_checks[name] = { start_after: start_after }
+    end
+
+    # Disables specific check
+    #
+    # For the list of available checks look at `lib/error_messages` module.
+    #
+    # @param name [Symbol] check name
+    # @return [void]
+    #
+    def disable_check(name)
+      enabled_checks.delete(name)
+    end
+
+    # Test whether specific check is enabled
+    #
+    # For the list of available checks look at `lib/error_messages` module.
+    #
+    # @param name [Symbol] check name
+    # @param version [Integer] migration version
+    # @return [void]
+    #
+    def check_enabled?(name, version: nil)
+      if enabled_checks[name]
+        start_after = enabled_checks[name][:start_after] || OnlineMigrations.config.start_after
+        !version || version > start_after
+      else
+        false
+      end
+    end
+
+    # Adds custom check
+    #
+    # @param start_after [Integer] migration version from which this check will be performed
+    #
+    # @yield [method, args] a block to be called with custom check
+    # @yieldparam method [Symbol] method name
+    # @yieldparam args [Array] method arguments
+    #
+    # @return [void]
+    #
+    # Use `stop!` method to stop the migration
+    #
+    # @example
+    #   OnlineMigrations.config.add_check do |method, args|
+    #     if method == :add_column && args[0].to_s == "users"
+    #       stop!("No more columns on the users table")
+    #     end
+    #   end
+    #
+    def add_check(start_after: nil, &block)
+      @checks << [{ start_after: start_after }, block]
+    end
+  end
+end

data/lib/online_migrations/copy_trigger.rb

@@ -0,0 +1,91 @@
+# frozen_string_literal: true
+
+require "openssl"
+
+module OnlineMigrations
+  # @private
+  class CopyTrigger
+    def self.on_table(table_name, connection:)
+      new(table_name, connection)
+    end
+
+    def name(from_columns, to_columns)
+      from_columns, to_columns = normalize_column_names(from_columns, to_columns)
+
+      joined_column_names = from_columns.zip(to_columns).flatten.join("_")
+      identifier = "#{table_name}_#{joined_column_names}"
+      hashed_identifier = OpenSSL::Digest::SHA256.hexdigest(identifier).first(10)
+      "trigger_#{hashed_identifier}"
+    end
+
+    def create(from_columns, to_columns)
+      from_columns, to_columns = normalize_column_names(from_columns, to_columns)
+      trigger_name = name(from_columns, to_columns)
+      assignment_clauses = assignment_clauses_for_columns(from_columns, to_columns)
+
+      connection.execute(<<~SQL)
+        CREATE OR REPLACE FUNCTION #{trigger_name}() RETURNS TRIGGER AS $$
+        BEGIN
+          #{assignment_clauses};
+          RETURN NEW;
+        END;
+        $$ LANGUAGE plpgsql;
+      SQL
+
+      connection.execute(<<~SQL)
+        DROP TRIGGER IF EXISTS #{trigger_name} ON #{quoted_table_name}
+      SQL
+
+      connection.execute(<<~SQL)
+        CREATE TRIGGER #{trigger_name}
+          BEFORE INSERT OR UPDATE
+          ON #{quoted_table_name}
+          FOR EACH ROW
+          EXECUTE PROCEDURE #{trigger_name}();
+      SQL
+    end
+
+    def remove(from_columns, to_columns)
+      trigger_name = name(from_columns, to_columns)
+
+      connection.execute("DROP TRIGGER IF EXISTS #{trigger_name} ON #{quoted_table_name}")
+      connection.execute("DROP FUNCTION IF EXISTS #{trigger_name}()")
+    end
+
+    private
+      attr_reader :table_name, :connection
+
+      def initialize(table_name, connection)
+        @table_name = table_name
+        @connection = connection
+      end
+
+      def quoted_table_name
+        @quoted_table_name ||= connection.quote_table_name(table_name)
+      end
+
+      def normalize_column_names(from_columns, to_columns)
+        from_columns = Array.wrap(from_columns)
+        to_columns = Array.wrap(to_columns)
+
+        if from_columns.size != to_columns.size
+          raise ArgumentError, "Number of source and destination columns must match"
+        end
+
+        [from_columns, to_columns]
+      end
+
+      def assignment_clauses_for_columns(from_columns, to_columns)
+        combined_column_names = to_columns.zip(from_columns)
+
+        assignment_clauses = combined_column_names.map do |(new_name, old_name)|
+          new_name = connection.quote_column_name(new_name)
+          old_name = connection.quote_column_name(old_name)
+
+          "NEW.#{new_name} := NEW.#{old_name}"
+        end
+
+        assignment_clauses.join(";\n  ")
+      end
+  end
+end

data/lib/online_migrations/database_tasks.rb

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+module OnlineMigrations
+  # @private
+  module DatabaseTasks
+    def migrate(*)
+      super
+    rescue => e # rubocop:disable Style/RescueStandardError
+      if e.cause.is_a?(OnlineMigrations::Error)
+        # strip cause
+        def e.cause
+          nil
+        end
+      end
+
+      raise e
+    end
+  end
+end