safe-pg-migrations 0.0.0 → 0.0.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: abc17a43f436f429bc2f8983a45b1a61af67a4f8
-  data.tar.gz: f1c0fdc3261c34fe99218e935e502c639d36f4f1
+  metadata.gz: 2a18b54460ec7f3977c9183273705d066793df42
+  data.tar.gz: ed2df6a14d6fc3a880b4629f10b6a22d3f6c105e
 SHA512:
-  metadata.gz: 3bc4cb7434a1b9059efbc0c69a7a3f73a329b478023db08b0f53d5345ad4346798be592b581e0db0413a574c6c72b2369dcad98402ae4f89a88411525967211d
-  data.tar.gz: 670b3ef74768301708b85e904defe0566dedfb04762ea85fb2cccf0dccb666a614719430e36e6204e399a7861c59494843750fb6321add67144a49c3fc941846
+  metadata.gz: 1d088c3c46622f6fa2f7cc717a8462f06042d3b9cbcd9ae8c0a79fb0f62f2fa1b1b47033050275b25d63add29b4f97795718aeb6dddbc3eae03f66b3614dfa62
+  data.tar.gz: d4754e18a023d4e38bfc8c4ae0a387c3e2c38646ba6bcdd776a0673983b885b93456654e27c0793c2841d512f6d40e0102878559b65d6b439dfa713a676b44b9

data/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2018 Doctolib
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

data/README.md

@@ -0,0 +1,17 @@
+# safe-pg-migrations
+
+[![Build Status](https://travis-ci.org/doctolib/safe-pg-migrations.svg?branch=master)](https://travis-ci.org/doctolib/safe-pg-migrations)
+
+## Compatibility
+
+Ruby 2.3+  
+Rails 5.2+  
+PostgreSQL 9.3+
+
+## Running tests
+
+```bash
+bundle
+psql -h localhost -c 'CREATE DATABASE safe_pg_migrations_test'
+rake test
+```

data/lib/safe-pg-migrations.rb

@@ -1,5 +1,3 @@
-class SafePgMigrations
-  def self.hello
-    puts 'Hello world!'
-  end
-end
+# frozen_string_literal: true
+
+require 'safe-pg-migrations/railtie' if defined?(Rails)

data/lib/safe-pg-migrations/base.rb

@@ -0,0 +1,83 @@
+# frozen_string_literal: true
+
+require 'safe-pg-migrations/configuration'
+require 'safe-pg-migrations/plugins/blocking_activity_logger'
+require 'safe-pg-migrations/plugins/statement_insurer'
+require 'safe-pg-migrations/plugins/statement_retrier'
+
+module SafePgMigrations
+  PLUGINS = [
+    BlockingActivityLogger,
+    StatementRetrier,
+    StatementInsurer,
+  ].freeze
+
+  class << self
+    attr_reader :current_migration
+    attr_accessor :enabled
+
+    def setup_and_teardown(migration, connection)
+      @alternate_connection = nil
+      @current_migration = migration
+      PLUGINS.each { |plugin| connection.extend(plugin) }
+      connection.with_setting(:lock_timeout, SafePgMigrations.config.safe_timeout) { yield }
+    ensure
+      close_alternate_connection
+      @current_migration = nil
+    end
+
+    def alternate_connection
+      @alternate_connection ||= ActiveRecord::Base.connection_pool.send(:new_connection)
+    end
+
+    def close_alternate_connection
+      return unless @alternate_connection
+
+      @alternate_connection.disconnect!
+      @alternate_connection = nil
+    end
+
+    def say(*args)
+      return unless current_migration
+
+      current_migration.say(*args)
+    end
+
+    def say_method_call(method, *args)
+      say "#{method}(#{args.map(&:inspect) * ', '})", true
+    end
+
+    def enabled?
+      return ENV['SAFE_PG_MIGRATIONS'] == '1' if ENV['SAFE_PG_MIGRATIONS']
+      return enabled unless enabled.nil?
+      return Rails.env.production? if defined?(Rails)
+
+      false
+    end
+
+    def config
+      @config ||= Configuration.new
+    end
+  end
+
+  module Migration
+    def exec_migration(connection, direction)
+      SafePgMigrations.setup_and_teardown(self, connection) do
+        super(connection, direction)
+      end
+    end
+
+    def disable_ddl_transaction
+      SafePgMigrations.enabled? || super
+    end
+
+    SAFE_METHODS = %i[execute add_column add_index add_reference add_belongs_to change_column_null].freeze
+    SAFE_METHODS.each do |method|
+      define_method method do |*args|
+        return super(*args) unless respond_to?(:safety_assured)
+
+        safety_assured { super(*args) }
+      end
+    end
+  end
+end

data/lib/safe-pg-migrations/configuration.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+require 'active_support/core_ext/numeric/time'
+
+module SafePgMigrations
+  class Configuration
+    attr_accessor :safe_timeout
+    attr_accessor :blocking_activity_logger_delay # Must be close to but smaller than safe_timeout.
+    attr_accessor :batch_size
+    attr_accessor :retry_delay
+    attr_accessor :max_tries
+
+    def initialize
+      self.safe_timeout = '5s'
+      self.blocking_activity_logger_delay = 4.seconds
+      self.batch_size = 1000
+      self.retry_delay = 2.minutes
+      self.max_tries = 5
+    end
+  end
+end

data/lib/safe-pg-migrations/plugins/blocking_activity_logger.rb

@@ -0,0 +1,74 @@
+# frozen_string_literal: true
+
+module SafePgMigrations
+  module BlockingActivityLogger
+    SELECT_BLOCKING_QUERIES_SQL = <<~SQL.squish
+      SELECT blocking_activity.query
+      FROM pg_catalog.pg_locks           blocked_locks
+      JOIN pg_catalog.pg_stat_activity   blocked_activity
+        ON blocked_activity.pid = blocked_locks.pid
+      JOIN pg_catalog.pg_locks           blocking_locks
+        ON blocking_locks.locktype = blocked_locks.locktype
+        AND blocking_locks.DATABASE      IS NOT DISTINCT FROM blocked_locks.DATABASE
+        AND blocking_locks.relation      IS NOT DISTINCT FROM blocked_locks.relation
+        AND blocking_locks.page          IS NOT DISTINCT FROM blocked_locks.page
+        AND blocking_locks.tuple         IS NOT DISTINCT FROM blocked_locks.tuple
+        AND blocking_locks.virtualxid    IS NOT DISTINCT FROM blocked_locks.virtualxid
+        AND blocking_locks.transactionid IS NOT DISTINCT FROM blocked_locks.transactionid
+        AND blocking_locks.classid       IS NOT DISTINCT FROM blocked_locks.classid
+        AND blocking_locks.objid         IS NOT DISTINCT FROM blocked_locks.objid
+        AND blocking_locks.objsubid      IS NOT DISTINCT FROM blocked_locks.objsubid
+        AND blocking_locks.pid != blocked_locks.pid
+      JOIN pg_catalog.pg_stat_activity   blocking_activity
+        ON blocking_activity.pid = blocking_locks.pid
+      WHERE blocked_locks.pid = %d
+    SQL
+
+    %i[
+      add_column remove_column add_foreign_key remove_foreign_key change_column_default
+      change_column_null create_table add_index remove_index
+    ].each do |method|
+      define_method method do |*args, &block|
+        log_blocking_queries { super(*args, &block) }
+      end
+    end
+
+    private
+
+    def log_blocking_queries
+      blocking_queries_retriever_thread =
+        Thread.new do
+          sleep SafePgMigrations.config.blocking_activity_logger_delay
+          SafePgMigrations.alternate_connection.query_values(SELECT_BLOCKING_QUERIES_SQL % raw_connection.backend_pid)
+        end
+
+      yield
+
+      blocking_queries_retriever_thread.kill
+    rescue ActiveRecord::LockWaitTimeout
+      SafePgMigrations.say 'Lock timeout.', true
+      queries =
+        begin
+          blocking_queries_retriever_thread.value
+        rescue StandardError => e
+          SafePgMigrations.say("Error while retrieving blocking queries: #{e}", true)
+          nil
+        end
+
+      raise if queries.nil?
+
+      if queries.empty?
+        SafePgMigrations.say 'Could not find any blocking query.', true
+      else
+        SafePgMigrations.say(
+          "Statement was being blocked by the following #{'query'.pluralize(queries.size)}:", true
+        )
+        SafePgMigrations.say '', true
+        queries.each { |query| SafePgMigrations.say "  #{query}", true }
+        SafePgMigrations.say '', true
+      end
+
+      raise
+    end
+  end
+end

data/lib/safe-pg-migrations/plugins/statement_insurer.rb

@@ -0,0 +1,90 @@
+# frozen_string_literal: true
+
+module SafePgMigrations
+  module StatementInsurer
+    %i[change_column_null add_foreign_key create_table].each do |method|
+      define_method method do |*args, &block|
+        with_setting(:statement_timeout, SafePgMigrations.config.safe_timeout) { super(*args, &block) }
+      end
+    end
+
+    def add_column(table_name, column_name, type, **options)
+      default = options.delete(:default)
+      null = options.delete(:null)
+
+      if !default.nil? || null == false
+        SafePgMigrations.say_method_call(:add_column, table_name, column_name, type, **options)
+      end
+
+      super
+
+      unless default.nil?
+        SafePgMigrations.say_method_call(:change_column_default, table_name, column_name, default)
+        change_column_default(table_name, column_name, default)
+
+        SafePgMigrations.say_method_call(:backfill_column_default, table_name, column_name)
+        backfill_column_default(table_name, column_name)
+      end
+
+      if null == false # rubocop:disable Style/GuardClause
+        SafePgMigrations.say_method_call(:change_column_null, table_name, column_name, null)
+        change_column_null(table_name, column_name, null)
+      end
+    end
+
+    def add_index(table_name, column_name, **options)
+      if SafePgMigrations.enabled?
+        options[:algorithm] = :concurrently
+        SafePgMigrations.say_method_call(:add_index, table_name, column_name, **options)
+      end
+      without_statement_timeout { super }
+    end
+
+    def remove_index(table_name, options = {})
+      options = { column: options } unless options.is_a?(Hash)
+      if SafePgMigrations.enabled?
+        options[:algorithm] = :concurrently
+        SafePgMigrations.say_method_call(:remove_index, table_name, **options)
+      end
+      without_statement_timeout { super }
+    end
+
+    def backfill_column_default(table_name, column_name)
+      quoted_table_name = quote_table_name(table_name)
+      quoted_column_name = quote_column_name(column_name)
+      primary_key_offset = 0
+      loop do
+        ids = query_values <<~SQL.squish
+          SELECT id FROM #{quoted_table_name} WHERE id > #{primary_key_offset}
+          ORDER BY id LIMIT #{SafePgMigrations.config.batch_size}
+        SQL
+        break if ids.empty?
+
+        primary_key_offset = ids.last
+        execute <<~SQL.squish
+          UPDATE #{quoted_table_name} SET #{quoted_column_name} = DEFAULT WHERE id IN (#{ids.join(',')})
+        SQL
+      end
+    end
+
+    def with_setting(key, value)
+      old_value = query_value("SHOW #{key}")
+      execute("SET #{key} TO #{quote(value)}")
+      begin
+        yield
+      ensure
+        begin
+          execute("SET #{key} TO #{quote(old_value)}")
+        rescue ActiveRecord::StatementInvalid => e
+          # Swallow `PG::InFailedSqlTransaction` exceptions so as to keep the
+          # original exception (if any).
+          raise unless e.cause.is_a?(PG::InFailedSqlTransaction)
+        end
+      end
+    end
+
+    def without_statement_timeout
+      with_setting(:statement_timeout, 0) { yield }
+    end
+  end
+end

data/lib/safe-pg-migrations/plugins/statement_retrier.rb

@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+module SafePgMigrations
+  module StatementRetrier
+    RETRIABLE_SCHEMA_STATEMENTS = %i[
+      add_column remove_column add_foreign_key remove_foreign_key change_column_default
+      change_column_null
+    ].freeze
+
+    RETRIABLE_SCHEMA_STATEMENTS.each do |method|
+      define_method method do |*args, &block|
+        retry_if_lock_timeout { super(*args, &block) }
+      end
+    end
+
+    private
+
+    def retry_if_lock_timeout
+      remaining_tries = SafePgMigrations.config.max_tries
+      begin
+        remaining_tries -= 1
+        yield
+      rescue ActiveRecord::LockWaitTimeout
+        raise if transaction_open? # Retrying is useless if we're inside a transaction.
+        raise unless remaining_tries > 0
+
+        retry_delay = SafePgMigrations.config.retry_delay
+        SafePgMigrations.say "Retrying in #{retry_delay} seconds...", true
+        sleep retry_delay
+        SafePgMigrations.say 'Retrying now.', true
+        retry
+      end
+    end
+  end
+end

data/lib/safe-pg-migrations/railtie.rb

@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+require 'safe-pg-migrations/base'
+
+module SafePgMigrations
+  class Railtie < Rails::Railtie
+    initializer 'sage_pg_migrations.insert_into_active_record' do
+      ActiveSupport.on_load :active_record do
+        ActiveRecord::Migration.prepend(SafePgMigrations::Migration)
+      end
+    end
+  end
+end

data/lib/safe-pg-migrations/version.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module SafePgMigrations
+  VERSION = '0.0.1'
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: safe-pg-migrations
 version: !ruby/object:Gem::Version
-  version: 0.0.0
+  version: 0.0.1
 platform: ruby
 authors:
 - Matthieu Prat
@@ -9,16 +9,165 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-10-08 00:00:00.000000000 Z
-dependencies: []
+date: 2018-10-10 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: activerecord
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '5.2'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '5.2'
+- !ruby/object:Gem::Dependency
+  name: activesupport
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '5.2'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '5.2'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: minitest
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: mocha
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: pg
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: pry
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: pry-coolline
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rubocop
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 description: Make your PG migrations safe.
 email: matthieuprat@gmail.com
 executables: []
 extensions: []
 extra_rdoc_files: []
 files:
+- LICENSE
+- README.md
 - lib/safe-pg-migrations.rb
-homepage: http://rubygems.org/doctolib/safe-pg-migrations
+- lib/safe-pg-migrations/base.rb
+- lib/safe-pg-migrations/configuration.rb
+- lib/safe-pg-migrations/plugins/blocking_activity_logger.rb
+- lib/safe-pg-migrations/plugins/statement_insurer.rb
+- lib/safe-pg-migrations/plugins/statement_retrier.rb
+- lib/safe-pg-migrations/railtie.rb
+- lib/safe-pg-migrations/version.rb
+homepage: https://github.com/doctolib/safe-pg-migrations
 licenses:
 - MIT
 metadata: {}
@@ -30,7 +179,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: '0'
+      version: '2.3'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
@@ -38,7 +187,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.6.13
+rubygems_version: 2.6.14.1
 signing_key: 
 specification_version: 4
 summary: Make your PG migrations safe.