activerecord-retry 1.0.0

data/lib/active_record/retry.rb

@@ -0,0 +1,89 @@
+require "active_record"
+require "active_record/errors"
+require "active_support/concern"
+require "active_support/core_ext/integer/inflections"
+require "active_support/core_ext/module/attribute_accessors"
+
+module ActiveRecord
+  module Retry
+    require "active_record/retry/version"
+
+    extend ActiveSupport::Concern
+
+    DEFAULT_RETRIES = [1, 2, 4].freeze
+    DEFAULT_RETRY_ERRORS = {
+      # MySQL errors
+      /Deadlock found when trying to get lock/ => :retry,
+      /Lock wait timeout exceeded/ => :retry,
+      /Lost connection to MySQL server during query/ => [:sleep, :reconnect, :retry],
+      /MySQL server has gone away/ => [:sleep, :reconnect, :retry],
+      /Query execution was interrupted/ => :retry,
+      /The MySQL server is running with the --read-only option so it cannot execute this statement/ => [:sleep, :reconnect, :retry]
+    }.freeze
+
+    included do
+      mattr_accessor :retry_errors, :retries
+
+      self.retries = self::DEFAULT_RETRIES.dup
+      self.retry_errors = self::DEFAULT_RETRY_ERRORS.dup
+
+      class << self
+        alias_method :find_by_sql_without_retry, :find_by_sql
+        alias_method :find_by_sql, :find_by_sql_with_retry
+        alias_method :transaction_without_retry, :transaction
+        alias_method :transaction, :transaction_with_retry
+      end
+    end
+
+    module ClassMethods
+      def find_by_sql_with_retry(*args, &block)
+        with_retry { find_by_sql_without_retry(*args, &block) }
+      end
+
+      def transaction_with_retry(*args, &block)
+        with_retry { transaction_without_retry(*args, &block) }
+      end
+
+      def with_retry
+        tries = 0
+
+        begin
+          yield
+        rescue ActiveRecord::StatementInvalid => error
+          raise if connection.open_transactions != 0
+          raise if tries >= retries.count
+
+          found, actions = retry_errors.detect { |regex, action| regex =~ error.message }
+          raise unless found
+
+          actions = Array(actions)
+          delay = retries[tries]
+          tries += 1
+
+          if logger
+            message = "Query failed: '#{error}'. "
+            message << actions.map do |action|
+              case action
+              when :sleep
+                "sleeping for #{delay}s"
+              when :reconnect
+                "reconnecting"
+              when :retry
+                "retrying"
+              end
+            end.join(", ").capitalize
+            message << " for the #{tries.ordinalize} time."
+            logger.warn(message)
+          end
+
+          sleep(delay) if actions.include?(:sleep)
+          if actions.include?(:reconnect)
+            clear_active_connections!
+            establish_connection
+          end
+          retry if actions.include?(:retry)
+        end
+      end
+    end
+  end
+end

data/lib/active_record/retry/rails.rb

@@ -0,0 +1,17 @@
+require "rails"
+require "active_record/retry"
+
+module ActiveRecord
+  module Retry
+    class Railtie < Rails::Railtie
+      config.active_record.retries = ActiveRecord::Retry::DEFAULT_RETRIES
+
+      config.after_initialize do |app|
+        ActiveSupport.on_load(:active_record) do
+          ActiveRecord::Base.send(:include, ActiveRecord::Retry)
+          ActiveRecord::Base.retries = app.config.active_record.retries
+        end
+      end
+    end
+  end
+end

data/lib/active_record/retry/version.rb

@@ -0,0 +1,5 @@
+module ActiveRecord
+  module Retry
+    VERSION = "1.0.0"
+  end
+end

data/lib/activerecord-retry.rb

@@ -0,0 +1 @@
+require "active_record/retry"

data/test/active_record/retry_test.rb

@@ -0,0 +1,122 @@
+require "test_helper"
+require "active_record/retry"
+
+describe ActiveRecord::Retry do
+  before do
+    @connection = mock()
+    @connection.singleton_class.class_exec do
+      attr_accessor :open_transactions
+    end
+    @connection.open_transactions = 0
+    @buffer = StringIO.new
+    @logger = Logger.new(@buffer)
+    @mock = Class.new
+    @mock.stubs(:connection).returns(@connection)
+    @mock.stubs(:clear_active_connections!)
+    @mock.stubs(:establish_connection)
+    @mock.stubs(:find_by_sql)
+    @mock.stubs(:logger).returns(@logger)
+    @mock.stubs(:sleep)
+    def @mock.transaction
+      connection.open_transactions += 1
+      yield
+    ensure
+      connection.open_transactions -= 1
+    end
+    @mock.send(:include, ActiveRecord::Retry)
+    @mock.retry_errors = {
+      /sleep then reconnect then retry/ => [:sleep, :reconnect, :retry],
+      /sleep then retry/ => [:sleep, :retry],
+      /reconnect then retry/ => [:reconnect, :retry],
+      /retry/ => :retry
+    }
+  end
+
+  it "should work with no errors" do
+    @mock.with_retry { :success }.must_equal(:success)
+  end
+
+  it "should work with less or equal errors than retries" do
+    errors = ["sleep then retry", "retry"]
+    @mock.retries = [0, 0]
+    @mock.with_retry { errors.any? ? raise(ActiveRecord::StatementInvalid, errors.shift) : :success }.must_equal(:success)
+  end
+
+  it "should not work with more errors than retries" do
+    errors = ["sleep then retry", "retry", "retry"]
+    @mock.retries = [0, 0]
+    -> { @mock.with_retry { errors.any? ? raise(ActiveRecord::StatementInvalid, errors.shift) : :success } }.must_raise(ActiveRecord::StatementInvalid)
+  end
+
+  it "should not retry more than retries count" do
+    retries = @mock.retries = [2, 4, 8, 16]
+    runs = 0
+
+    -> do
+      @mock.with_retry do
+        runs += 1
+        raise ActiveRecord::StatementInvalid, "retry"
+      end
+    end.must_raise(ActiveRecord::StatementInvalid)
+
+    runs.must_equal(1 + retries.count)
+  end
+
+  it "should not retry a query inside of a nested transaction" do
+    inner_runs = outer_runs = 0
+
+    -> do
+      @mock.transaction do
+        outer_runs += 1
+        @mock.with_retry do
+          inner_runs += 1
+          raise ActiveRecord::StatementInvalid, "retry"
+        end
+      end
+    end.must_raise(ActiveRecord::StatementInvalid)
+
+    inner_runs.must_equal(outer_runs)
+  end
+
+  it "logs a warning when a query is being retried that describes what it is doing" do
+    @mock.retries = [2]
+
+    -> do
+      @mock.with_retry do
+        raise ActiveRecord::StatementInvalid, "sleep then reconnect then retry"
+      end
+    end.must_raise(ActiveRecord::StatementInvalid)
+
+    warning = @buffer.string
+    warning.wont_be_empty
+    /sleeping for 2s/i.must_match(warning)
+    /reconnecting/i.must_match(warning)
+    /retrying/i.must_match(warning)
+    /for the 1st time/i.must_match(warning)
+  end
+
+  it "sleeps for the proper times" do
+    @mock.expects(:sleep).with(2)
+    @mock.expects(:sleep).with(4)
+    @mock.expects(:sleep).with(8)
+    @mock.retries = [2, 4, 8]
+
+    -> do
+      @mock.with_retry do
+        raise ActiveRecord::StatementInvalid, "sleep then retry"
+      end
+    end.must_raise(ActiveRecord::StatementInvalid)
+  end
+
+  it "clears active connections when action is reconnect" do
+    @mock.expects(:clear_active_connections!)
+    @mock.expects(:establish_connection)
+    @mock.retries = [2]
+
+    -> do
+      @mock.with_retry do
+        raise ActiveRecord::StatementInvalid, "reconnect then retry"
+      end
+    end.must_raise(ActiveRecord::StatementInvalid)
+  end
+end

metadata

@@ -0,0 +1,138 @@
+--- !ruby/object:Gem::Specification
+name: activerecord-retry
+version: !ruby/object:Gem::Version
+  version: 1.0.0
+  prerelease: 
+platform: ruby
+authors:
+- Samuel Kadolph
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2012-09-06 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: activerecord
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '3.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '3.0'
+- !ruby/object:Gem::Dependency
+  name: activesupport
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '3.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '3.0'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 1.1.5
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 1.1.5
+- !ruby/object:Gem::Dependency
+  name: mocha
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.12.1
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.12.1
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.9.2.2
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.9.2.2
+description: Retries reads and transactions when an ActiveRecord::StatementInvalid
+  occurs that matches a list of errors. Default list of errors includes errors related
+  to failover situations to allow for graceful failovers during a request and to attempt
+  prevention of data loss for temporary issue.
+email:
+- samuel@kadolph.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- lib/active_record/retry/rails.rb
+- lib/active_record/retry/version.rb
+- lib/active_record/retry.rb
+- lib/activerecord-retry.rb
+- test/active_record/retry_test.rb
+homepage: http://samuelkadolph.github.com/activerecord-retry/
+licenses: []
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: 1.9.2
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+      segments:
+      - 0
+      hash: -2741769345565936512
+requirements: []
+rubyforge_project: 
+rubygems_version: 1.8.24
+signing_key: 
+specification_version: 3
+summary: activerecord-retry add query retrying to ActiveRecord reads and transactions
+  on specific errors.
+test_files:
+- test/active_record/retry_test.rb