pg_failover 1.0.0.pre.alpha

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 940fc1e1a1faf10f7dbc18910a51f53a724b276dc5ad4e377251bc205dcf3291
+  data.tar.gz: 7b888845de98b57b6724cc0144c946ec27df321d02e025310dd7c04ed2a87cdf
+SHA512:
+  metadata.gz: 32d0f7a5aee56766df278c53bba22ab439ec5914f96cd4c4aaeca97c5279aa9447fdfb1457f77015d15ca280f74ab01a2e17d8b7972660e6fc50618e7f512478
+  data.tar.gz: e0dc8a6d86ab6c6902dd37435a601a7095a4da366ce9da69001ead7ad3e257ae693d7ea66b82a051b6eb459bffe4cac525e43a705ca57eb8bcace829b9f5c962

data/.circleci/config.yml

@@ -0,0 +1,104 @@
+version: 2.1
+commands:
+  install-dependencies:
+    steps:
+      - restore_cache:
+          keys:
+            - pg_failover-{{ checksum "pg_failover.gemspec" }}-{{ checksum "Gemfile" }}
+      - run:
+          name: Install Ruby gems
+          command: bundle check --path=vendor/bundle || bundle install --path=vendor/bundle
+      - save_cache:
+          key: pg_failover-{{ checksum "pg_failover.gemspec" }}-{{ checksum "Gemfile" }}
+          paths:
+            - vendor/bundle
+  test:
+    steps:
+      - run:
+          name: Run unit tests
+          command: bundle exec rspec
+
+executors:
+  ruby:
+    docker:
+      - image: circleci/ruby:<< parameters.tag >>
+    parameters:
+      tag:
+        description: "Docker image tag"
+        default: "latest"
+        type: string
+
+jobs:
+  build:
+    executor:
+      name: ruby
+      tag: << parameters.tag >>
+    parameters:
+      tag:
+        description: "Docker image tag"
+        default: << parameters.tag >>
+        type: string
+    steps:
+      - checkout
+      - install-dependencies
+      - test
+  publish:
+    executor:
+      name: ruby
+    steps:
+      - checkout
+      - install-dependencies
+      - run:
+          name: Build gem
+          command: bundle exec rake build
+      - run:
+          name: Configure RubyGems API key
+          command: |
+            mkdir -p ~/.gem
+            echo ":rubygems_api_key: $RUBYGEMS_API_KEY" >  ~/.gem/credentials
+            chmod 0600 ~/.gem/credentials
+      - run:
+          name: Publish gem
+          command: |
+            package=$(ls -t1 pkg/pg_failover-*.gem | head -1)
+            gem push "$package"
+
+workflows:
+  test-and-publish:
+    jobs:
+      - build:
+          filters:
+            tags:
+              only: /.*/
+          name: "ruby23"
+          tag: "2.3"
+      - build:
+          filters:
+            tags:
+              only: /.*/
+          name: "ruby24"
+          tag: "2.4"
+      - build:
+          filters:
+            tags:
+              only: /.*/
+          name: "ruby25"
+          tag: "2.5"
+      - build:
+          filters:
+            tags:
+              only: /.*/
+          name: "ruby26"
+          tag: "2.6"
+      - publish:
+          context: org-rubygems
+          filters:
+            branches:
+              ignore: /.*/
+            tags:
+              only: /^v\d{1,2}\.\d{1,2}\.\d{1,2}.*/
+          requires:
+            - ruby23
+            - ruby24
+            - ruby25
+            - ruby26

data/.gitignore

@@ -0,0 +1,10 @@
+/.bundle/
+/.rspec_status
+/.yardoc
+/Gemfile.lock
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/

data/.rspec

@@ -0,0 +1,3 @@
+--format documentation
+--color
+--require spec_helper

data/CHANGELOG.md

@@ -0,0 +1 @@
+v1.0.0 Initial release.

data/Gemfile

@@ -0,0 +1,9 @@
+source "https://rubygems.org"
+
+group :development, :test do
+  gem 'activerecord'
+  gem 'pg'
+  gem 'sequel'
+end
+
+gemspec

data/LICENSE

@@ -0,0 +1,27 @@
+Copyright © 2019, Funding Circle.
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+1. Redistributions of source code must retain the above copyright notice, this
+list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright notice,
+this list of conditions and the following disclaimer in the documentation
+and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its contributors
+may be used to endorse or promote products derived from this software without
+specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

data/README.md

@@ -0,0 +1,80 @@
+# PgFailover
+
+Handle potential failover events in PostgreSQL database connections by
+reconnecting if the database is in a recovery mode. This check occurs when a
+connection is checked out of the connection pool.
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'pg_failover'
+```
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install pg_failover
+
+## Usage
+
+This library can be configured via a code block or environment variables.
+
+The configuration aspects are as follows:
+- Enabled - Hooks a callback to the `pg` adapter.
+- Max retries - How many times an attempt to reconnect should be made.
+- Throttle interval - The period between checks on database connections.
+
+In an initializer:
+
+```ruby
+PgFailover.configure do |config|
+  config.enabled = true
+  config.max_retries = 3
+  config.throttle_interval = 3.3
+end
+```
+
+You can also configure the logging device to be used:
+
+```ruby
+PgFailover.configure do |config|
+  config.logger = Rails.logger
+end
+```
+
+Using environment variables:
+
+    POSTGRES_FAILOVER_ENABLED=true
+    POSTGRES_FAILOVER_MAX_RETRIES=3
+    POSTGRES_FAILOVER_THROTTLE_INTERVAL=3.3
+
+The above settings demonstrate enabling the failover checks and will only attempt to
+re-establish a database connection 3 times with a pause of 3.3 seconds between
+each attempt.
+
+## Development
+
+After checking out the repo, run `bin/setup` to install dependencies. Then, run
+`rake test` to run the tests. You can also run `bin/console` for an interactive
+prompt that will allow you to experiment.
+
+To install this gem onto your local machine, run `bundle exec rake install`. To
+release a new version, update the version number in `pg_failover.gemspec`, and
+then run `bundle exec rake release`, which will create a git tag for the
+version, push git commits and tags, and push the `.gem` file to
+[rubygems.org](https://rubygems.org).
+
+## Contributing
+
+Bug reports and pull requests are welcome on GitHub at https://github.com/FundingCircle/pg_failover.
+
+## License
+
+Copyright © 2019 Funding Circle.
+
+Distributed under the BSD 3-Clause License.

data/Rakefile

@@ -0,0 +1,6 @@
+require "bundler/gem_tasks"
+require "rspec/core/rake_task"
+
+RSpec::Core::RakeTask.new(:spec)
+
+task :default => :spec

data/bin/console

@@ -0,0 +1,7 @@
+#!/usr/bin/env ruby
+
+require "bundler/setup"
+require "pg_failover"
+require "irb"
+
+IRB.start(__FILE__)

data/bin/setup

@@ -0,0 +1,6 @@
+#!/usr/bin/env bash
+set -euo pipefail
+IFS=$'\n\t'
+set -vx
+
+bundle install

data/lib/pg_failover/active_record_adapter.rb

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+module PgFailover
+  class ActiveRecordAdapter
+    def self.enable
+      ::ActiveRecord::ConnectionAdapters::PostgreSQLAdapter.set_callback(:checkout, :after) do
+        # this seems to be the connection pool that internaly has the connection loaded for the current thread
+        # methods like #execute and #reconnect! are against that connection
+        connection = self
+
+        PgFailover.connection_validator.call(
+          throttle_by: connection.raw_connection,
+          in_recovery: proc {
+            result = connection.execute('select pg_is_in_recovery()').first
+
+            %w[1 t true].include?(result['pg_is_in_recovery'].to_s)
+          },
+          reconnect: proc { connection.reconnect! }
+        )
+      end
+    end
+  end
+end

data/lib/pg_failover/config.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+module PgFailover
+  Config = Struct.new(:enabled, :logger, :max_retries, :throttle_interval) do
+    def logger
+      self[:logger] ||= Logger.new($stdout)
+    end
+
+    def throttle_interval
+      self[:throttle_interval] ||= (ENV['POSTGRES_FAILOVER_THROTTLE_INTERVAL'] || 10.0).to_f
+    end
+
+    def throttle_enabled?
+      !throttle_interval.zero?
+    end
+
+    def max_retries
+      self[:max_retries] ||= (ENV['POSTGRES_FAILOVER_MAX_RETRIES'] || 1).to_i
+    end
+
+    def enabled
+      self[:enabled] ||= %w(1 t true).include?(ENV['POSTGRES_FAILOVER_ENABLED'])
+    end
+
+    def enabled?
+      !!enabled
+    end
+  end
+end

data/lib/pg_failover/connection_validator.rb

@@ -0,0 +1,48 @@
+# frozen_string_literal: true
+
+module PgFailover
+  class ConnectionValidator
+    def initialize(config)
+      @logger = config.logger
+      @max_retries = config.max_retries
+      @throttle = Throttle.new(throttle_interval: config.throttle_interval)
+      @config = config
+    end
+
+    attr_reader :logger, :max_retries, :throttle, :throttle_interval, :config
+
+    def call(in_recovery:, reconnect:, throttle_by:)
+      if config.throttle_enabled?
+        throttle.on_stale(throttle_by) { check_and_reconnect(in_recovery, reconnect) }
+      else
+        check_and_reconnect(in_recovery, reconnect)
+      end
+    end
+
+    private
+
+    def check_if_db_is_in_recovery(in_recovery)
+      in_recovery.call
+    rescue StandardError => e
+      logger.error("Got an error while trying to check pg_is_in_recovery, #{e.class}\n#{e.message}")
+      true
+    end
+
+    def check_and_reconnect(in_recovery, reconnect)
+      reconnect_attempts = 0
+
+      while (connection_in_recovery = check_if_db_is_in_recovery(in_recovery))
+        logger.info("The database is in recovery. Trying to reconnect. Attempt #{reconnect_attempts} from #{@max_retries}")
+        reconnect.call
+
+        reconnect_attempts += 1
+
+        break if reconnect_attempts >= @max_retries
+
+        sleep(rand(0..0.2))
+      end
+
+      !connection_in_recovery
+    end
+  end
+end

data/lib/pg_failover/sequel_adapter.rb

@@ -0,0 +1,51 @@
+# frozen_string_literal: true
+
+module PgFailover
+  module SequelAdapter
+    class << self
+      def register_extension
+        ::Sequel::Database.register_extension(:postgres_failover_validator) do |db|
+          db.pool.extend(PgFailover::SequelAdapter::ConnectionValidator)
+        end
+      end
+
+      def enable(databases = Sequel::DATABASES)
+        register_extension
+
+        databases.each do |db|
+          db.extension :postgres_failover_validator if db.adapter_scheme == :postgres
+        end
+      end
+    end
+
+    module ConnectionValidator
+      def acquire(*a)
+        connection = super
+
+        PgFailover.connection_validator.call(
+          throttle_by: connection,
+          in_recovery: proc {
+            result = connection.execute('select pg_is_in_recovery()') { |r| r.to_a }.first
+            %w[1 t true].include?(result['pg_is_in_recovery'].to_s)
+          },
+          reconnect: proc {
+            # This disconnect is copy pasted from the
+            # https://github.com/jeremyevans/sequel/blob/5.15.0/lib/sequel/extensions/connection_validator.rb#L103-L109
+            #
+            if pool_type == :sharded_threaded
+              sync{allocated(a.last).delete(Thread.current)}
+            else
+              sync{@allocated.delete(Thread.current)}
+            end
+
+            disconnect_connection(connection)
+
+            connection = super
+          }
+        )
+
+        connection
+      end
+    end
+  end
+end

data/lib/pg_failover/throttle.rb

@@ -0,0 +1,45 @@
+# frozen_string_literal: true
+
+module PgFailover
+  class Throttle
+    def initialize(throttle_interval: nil)
+      @last_good_at = {}
+      @throttle_interval = throttle_interval || 0.0
+    end
+
+    attr_reader :throttle_interval
+
+    def size
+      @last_good_at.count
+    end
+
+    def on_stale(connection)
+      return if should_throttle?(connection)
+
+      clear_stale_throttle_times
+
+      valid = yield
+
+      @last_good_at[connection] = Time.now.to_f if valid
+    end
+
+    def should_throttle?(connection)
+      return if @last_good_at[connection].nil?
+
+      connection_check_age = (Time.now.to_f - @last_good_at[connection])
+      connection_check_age < throttle_interval
+    end
+
+    def known?(connection)
+      @last_good_at[connection]
+    end
+
+    private
+
+    def clear_stale_throttle_times
+      stale_after = Time.now.to_f - throttle_interval * 3
+
+      @last_good_at.delete_if { |_k, v| stale_after > v }
+    end
+  end
+end

data/lib/pg_failover.rb

@@ -0,0 +1,35 @@
+module PgFailover
+  autoload :ActiveRecordAdapter, 'pg_failover/active_record_adapter'
+  autoload :Config, 'pg_failover/config'
+  autoload :ConnectionValidator,  'pg_failover/connection_validator'
+  autoload :SequelAdapter, 'pg_failover/sequel_adapter'
+  autoload :Throttle, 'pg_failover/throttle'
+
+  class << self
+    def configure
+      yield configuration if block_given?
+
+      if configuration.enabled?
+        if configuration.throttle_enabled?
+          configuration.logger.info("Enabled PgFailover policy (one check per #{configuration.throttle_interval} seconds per connection on checkout)")
+        else
+          configuration.logger.info('Enabled PgFailover policy (one check for every checkout from the connection pool)')
+        end
+
+        SequelAdapter.enable if defined?(::Sequel)
+        ActiveRecordAdapter.enable if defined?(::ActiveRecord::ConnectionAdapters::PostgreSQLAdapter)
+
+      else
+        configuration.logger.warn 'Disabled PgFailover policy'
+      end
+    end
+
+    def connection_validator
+      @connection_validator ||= ConnectionValidator.new(configuration)
+    end
+
+    def configuration
+      @configuration ||= Config.new
+    end
+  end
+end

data/pg_failover.gemspec

@@ -0,0 +1,34 @@
+Gem::Specification.new do |spec|
+  spec.name          = "pg_failover"
+  spec.version       = "1.0.0-alpha"
+  spec.authors       = ["Aleksandar Ivanov", "Andy Chambers", "Sasha Gerrand"]
+  spec.email         = ["engineering+pg_failover@fundingcircle.com"]
+  spec.license       = "BSD-3-Clause"
+
+  spec.summary       = %q{Handle Postgres failover events gracefully.}
+  spec.description   = %q{Handle Postgres failover events gracefully using your favourite ORM.}
+  spec.homepage      = "https://github.com/FundingCircle/pg_failover"
+
+  # Prevent pushing this gem to RubyGems.org. To allow pushes either set the 'allowed_push_host'
+  # to allow pushing to a single host or delete this section to allow pushing to any host.
+  raise "RubyGems 2.0 or newer is required to protect against public gem pushes." unless spec.respond_to?(:metadata)
+
+  spec.metadata = {
+    "changelog_uri"   => "https://github.com/FundingCircle/pg_failover/blob/master/CHANGELOG.md",
+    "homepage_uri"    => spec.homepage,
+    "source_code_uri" => "https://github.com/FundingCircle/pg_failover",
+  }
+
+  # Specify which files should be added to the gem when it is released.
+  # The `git ls-files -z` loads the files in the RubyGem that have been added into git.
+  spec.files         = Dir.chdir(File.expand_path('..', __FILE__)) do
+    `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
+  end
+  spec.bindir        = "exe"
+  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.require_paths = ["lib"]
+
+  spec.add_development_dependency "bundler", "~> 1.17"
+  spec.add_development_dependency "rake", "~> 10.0"
+  spec.add_development_dependency "rspec", "~> 3.0"
+end

metadata

@@ -0,0 +1,107 @@
+--- !ruby/object:Gem::Specification
+name: pg_failover
+version: !ruby/object:Gem::Version
+  version: 1.0.0.pre.alpha
+platform: ruby
+authors:
+- Aleksandar Ivanov
+- Andy Chambers
+- Sasha Gerrand
+autorequire: 
+bindir: exe
+cert_chain: []
+date: 2019-02-06 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.17'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.17'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+description: Handle Postgres failover events gracefully using your favourite ORM.
+email:
+- engineering+pg_failover@fundingcircle.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".circleci/config.yml"
+- ".gitignore"
+- ".rspec"
+- CHANGELOG.md
+- Gemfile
+- LICENSE
+- README.md
+- Rakefile
+- bin/console
+- bin/setup
+- lib/pg_failover.rb
+- lib/pg_failover/active_record_adapter.rb
+- lib/pg_failover/config.rb
+- lib/pg_failover/connection_validator.rb
+- lib/pg_failover/sequel_adapter.rb
+- lib/pg_failover/throttle.rb
+- pg_failover.gemspec
+homepage: https://github.com/FundingCircle/pg_failover
+licenses:
+- BSD-3-Clause
+metadata:
+  changelog_uri: https://github.com/FundingCircle/pg_failover/blob/master/CHANGELOG.md
+  homepage_uri: https://github.com/FundingCircle/pg_failover
+  source_code_uri: https://github.com/FundingCircle/pg_failover
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">"
+    - !ruby/object:Gem::Version
+      version: 1.3.1
+requirements: []
+rubygems_version: 3.0.1
+signing_key: 
+specification_version: 4
+summary: Handle Postgres failover events gracefully.
+test_files: []