activerecord-retry 1.0.0
Sign up to get free protection for your applications and to get access to all the features.
- data/lib/active_record/retry.rb +89 -0
- data/lib/active_record/retry/rails.rb +17 -0
- data/lib/active_record/retry/version.rb +5 -0
- data/lib/activerecord-retry.rb +1 -0
- data/test/active_record/retry_test.rb +122 -0
- metadata +138 -0
@@ -0,0 +1,89 @@
|
|
1
|
+
require "active_record"
|
2
|
+
require "active_record/errors"
|
3
|
+
require "active_support/concern"
|
4
|
+
require "active_support/core_ext/integer/inflections"
|
5
|
+
require "active_support/core_ext/module/attribute_accessors"
|
6
|
+
|
7
|
+
module ActiveRecord
|
8
|
+
module Retry
|
9
|
+
require "active_record/retry/version"
|
10
|
+
|
11
|
+
extend ActiveSupport::Concern
|
12
|
+
|
13
|
+
DEFAULT_RETRIES = [1, 2, 4].freeze
|
14
|
+
DEFAULT_RETRY_ERRORS = {
|
15
|
+
# MySQL errors
|
16
|
+
/Deadlock found when trying to get lock/ => :retry,
|
17
|
+
/Lock wait timeout exceeded/ => :retry,
|
18
|
+
/Lost connection to MySQL server during query/ => [:sleep, :reconnect, :retry],
|
19
|
+
/MySQL server has gone away/ => [:sleep, :reconnect, :retry],
|
20
|
+
/Query execution was interrupted/ => :retry,
|
21
|
+
/The MySQL server is running with the --read-only option so it cannot execute this statement/ => [:sleep, :reconnect, :retry]
|
22
|
+
}.freeze
|
23
|
+
|
24
|
+
included do
|
25
|
+
mattr_accessor :retry_errors, :retries
|
26
|
+
|
27
|
+
self.retries = self::DEFAULT_RETRIES.dup
|
28
|
+
self.retry_errors = self::DEFAULT_RETRY_ERRORS.dup
|
29
|
+
|
30
|
+
class << self
|
31
|
+
alias_method :find_by_sql_without_retry, :find_by_sql
|
32
|
+
alias_method :find_by_sql, :find_by_sql_with_retry
|
33
|
+
alias_method :transaction_without_retry, :transaction
|
34
|
+
alias_method :transaction, :transaction_with_retry
|
35
|
+
end
|
36
|
+
end
|
37
|
+
|
38
|
+
module ClassMethods
|
39
|
+
def find_by_sql_with_retry(*args, &block)
|
40
|
+
with_retry { find_by_sql_without_retry(*args, &block) }
|
41
|
+
end
|
42
|
+
|
43
|
+
def transaction_with_retry(*args, &block)
|
44
|
+
with_retry { transaction_without_retry(*args, &block) }
|
45
|
+
end
|
46
|
+
|
47
|
+
def with_retry
|
48
|
+
tries = 0
|
49
|
+
|
50
|
+
begin
|
51
|
+
yield
|
52
|
+
rescue ActiveRecord::StatementInvalid => error
|
53
|
+
raise if connection.open_transactions != 0
|
54
|
+
raise if tries >= retries.count
|
55
|
+
|
56
|
+
found, actions = retry_errors.detect { |regex, action| regex =~ error.message }
|
57
|
+
raise unless found
|
58
|
+
|
59
|
+
actions = Array(actions)
|
60
|
+
delay = retries[tries]
|
61
|
+
tries += 1
|
62
|
+
|
63
|
+
if logger
|
64
|
+
message = "Query failed: '#{error}'. "
|
65
|
+
message << actions.map do |action|
|
66
|
+
case action
|
67
|
+
when :sleep
|
68
|
+
"sleeping for #{delay}s"
|
69
|
+
when :reconnect
|
70
|
+
"reconnecting"
|
71
|
+
when :retry
|
72
|
+
"retrying"
|
73
|
+
end
|
74
|
+
end.join(", ").capitalize
|
75
|
+
message << " for the #{tries.ordinalize} time."
|
76
|
+
logger.warn(message)
|
77
|
+
end
|
78
|
+
|
79
|
+
sleep(delay) if actions.include?(:sleep)
|
80
|
+
if actions.include?(:reconnect)
|
81
|
+
clear_active_connections!
|
82
|
+
establish_connection
|
83
|
+
end
|
84
|
+
retry if actions.include?(:retry)
|
85
|
+
end
|
86
|
+
end
|
87
|
+
end
|
88
|
+
end
|
89
|
+
end
|
@@ -0,0 +1,17 @@
|
|
1
|
+
require "rails"
|
2
|
+
require "active_record/retry"
|
3
|
+
|
4
|
+
module ActiveRecord
|
5
|
+
module Retry
|
6
|
+
class Railtie < Rails::Railtie
|
7
|
+
config.active_record.retries = ActiveRecord::Retry::DEFAULT_RETRIES
|
8
|
+
|
9
|
+
config.after_initialize do |app|
|
10
|
+
ActiveSupport.on_load(:active_record) do
|
11
|
+
ActiveRecord::Base.send(:include, ActiveRecord::Retry)
|
12
|
+
ActiveRecord::Base.retries = app.config.active_record.retries
|
13
|
+
end
|
14
|
+
end
|
15
|
+
end
|
16
|
+
end
|
17
|
+
end
|
@@ -0,0 +1 @@
|
|
1
|
+
require "active_record/retry"
|
@@ -0,0 +1,122 @@
|
|
1
|
+
require "test_helper"
|
2
|
+
require "active_record/retry"
|
3
|
+
|
4
|
+
describe ActiveRecord::Retry do
|
5
|
+
before do
|
6
|
+
@connection = mock()
|
7
|
+
@connection.singleton_class.class_exec do
|
8
|
+
attr_accessor :open_transactions
|
9
|
+
end
|
10
|
+
@connection.open_transactions = 0
|
11
|
+
@buffer = StringIO.new
|
12
|
+
@logger = Logger.new(@buffer)
|
13
|
+
@mock = Class.new
|
14
|
+
@mock.stubs(:connection).returns(@connection)
|
15
|
+
@mock.stubs(:clear_active_connections!)
|
16
|
+
@mock.stubs(:establish_connection)
|
17
|
+
@mock.stubs(:find_by_sql)
|
18
|
+
@mock.stubs(:logger).returns(@logger)
|
19
|
+
@mock.stubs(:sleep)
|
20
|
+
def @mock.transaction
|
21
|
+
connection.open_transactions += 1
|
22
|
+
yield
|
23
|
+
ensure
|
24
|
+
connection.open_transactions -= 1
|
25
|
+
end
|
26
|
+
@mock.send(:include, ActiveRecord::Retry)
|
27
|
+
@mock.retry_errors = {
|
28
|
+
/sleep then reconnect then retry/ => [:sleep, :reconnect, :retry],
|
29
|
+
/sleep then retry/ => [:sleep, :retry],
|
30
|
+
/reconnect then retry/ => [:reconnect, :retry],
|
31
|
+
/retry/ => :retry
|
32
|
+
}
|
33
|
+
end
|
34
|
+
|
35
|
+
it "should work with no errors" do
|
36
|
+
@mock.with_retry { :success }.must_equal(:success)
|
37
|
+
end
|
38
|
+
|
39
|
+
it "should work with less or equal errors than retries" do
|
40
|
+
errors = ["sleep then retry", "retry"]
|
41
|
+
@mock.retries = [0, 0]
|
42
|
+
@mock.with_retry { errors.any? ? raise(ActiveRecord::StatementInvalid, errors.shift) : :success }.must_equal(:success)
|
43
|
+
end
|
44
|
+
|
45
|
+
it "should not work with more errors than retries" do
|
46
|
+
errors = ["sleep then retry", "retry", "retry"]
|
47
|
+
@mock.retries = [0, 0]
|
48
|
+
-> { @mock.with_retry { errors.any? ? raise(ActiveRecord::StatementInvalid, errors.shift) : :success } }.must_raise(ActiveRecord::StatementInvalid)
|
49
|
+
end
|
50
|
+
|
51
|
+
it "should not retry more than retries count" do
|
52
|
+
retries = @mock.retries = [2, 4, 8, 16]
|
53
|
+
runs = 0
|
54
|
+
|
55
|
+
-> do
|
56
|
+
@mock.with_retry do
|
57
|
+
runs += 1
|
58
|
+
raise ActiveRecord::StatementInvalid, "retry"
|
59
|
+
end
|
60
|
+
end.must_raise(ActiveRecord::StatementInvalid)
|
61
|
+
|
62
|
+
runs.must_equal(1 + retries.count)
|
63
|
+
end
|
64
|
+
|
65
|
+
it "should not retry a query inside of a nested transaction" do
|
66
|
+
inner_runs = outer_runs = 0
|
67
|
+
|
68
|
+
-> do
|
69
|
+
@mock.transaction do
|
70
|
+
outer_runs += 1
|
71
|
+
@mock.with_retry do
|
72
|
+
inner_runs += 1
|
73
|
+
raise ActiveRecord::StatementInvalid, "retry"
|
74
|
+
end
|
75
|
+
end
|
76
|
+
end.must_raise(ActiveRecord::StatementInvalid)
|
77
|
+
|
78
|
+
inner_runs.must_equal(outer_runs)
|
79
|
+
end
|
80
|
+
|
81
|
+
it "logs a warning when a query is being retried that describes what it is doing" do
|
82
|
+
@mock.retries = [2]
|
83
|
+
|
84
|
+
-> do
|
85
|
+
@mock.with_retry do
|
86
|
+
raise ActiveRecord::StatementInvalid, "sleep then reconnect then retry"
|
87
|
+
end
|
88
|
+
end.must_raise(ActiveRecord::StatementInvalid)
|
89
|
+
|
90
|
+
warning = @buffer.string
|
91
|
+
warning.wont_be_empty
|
92
|
+
/sleeping for 2s/i.must_match(warning)
|
93
|
+
/reconnecting/i.must_match(warning)
|
94
|
+
/retrying/i.must_match(warning)
|
95
|
+
/for the 1st time/i.must_match(warning)
|
96
|
+
end
|
97
|
+
|
98
|
+
it "sleeps for the proper times" do
|
99
|
+
@mock.expects(:sleep).with(2)
|
100
|
+
@mock.expects(:sleep).with(4)
|
101
|
+
@mock.expects(:sleep).with(8)
|
102
|
+
@mock.retries = [2, 4, 8]
|
103
|
+
|
104
|
+
-> do
|
105
|
+
@mock.with_retry do
|
106
|
+
raise ActiveRecord::StatementInvalid, "sleep then retry"
|
107
|
+
end
|
108
|
+
end.must_raise(ActiveRecord::StatementInvalid)
|
109
|
+
end
|
110
|
+
|
111
|
+
it "clears active connections when action is reconnect" do
|
112
|
+
@mock.expects(:clear_active_connections!)
|
113
|
+
@mock.expects(:establish_connection)
|
114
|
+
@mock.retries = [2]
|
115
|
+
|
116
|
+
-> do
|
117
|
+
@mock.with_retry do
|
118
|
+
raise ActiveRecord::StatementInvalid, "reconnect then retry"
|
119
|
+
end
|
120
|
+
end.must_raise(ActiveRecord::StatementInvalid)
|
121
|
+
end
|
122
|
+
end
|
metadata
ADDED
@@ -0,0 +1,138 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: activerecord-retry
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 1.0.0
|
5
|
+
prerelease:
|
6
|
+
platform: ruby
|
7
|
+
authors:
|
8
|
+
- Samuel Kadolph
|
9
|
+
autorequire:
|
10
|
+
bindir: bin
|
11
|
+
cert_chain: []
|
12
|
+
date: 2012-09-06 00:00:00.000000000 Z
|
13
|
+
dependencies:
|
14
|
+
- !ruby/object:Gem::Dependency
|
15
|
+
name: activerecord
|
16
|
+
requirement: !ruby/object:Gem::Requirement
|
17
|
+
none: false
|
18
|
+
requirements:
|
19
|
+
- - ~>
|
20
|
+
- !ruby/object:Gem::Version
|
21
|
+
version: '3.0'
|
22
|
+
type: :runtime
|
23
|
+
prerelease: false
|
24
|
+
version_requirements: !ruby/object:Gem::Requirement
|
25
|
+
none: false
|
26
|
+
requirements:
|
27
|
+
- - ~>
|
28
|
+
- !ruby/object:Gem::Version
|
29
|
+
version: '3.0'
|
30
|
+
- !ruby/object:Gem::Dependency
|
31
|
+
name: activesupport
|
32
|
+
requirement: !ruby/object:Gem::Requirement
|
33
|
+
none: false
|
34
|
+
requirements:
|
35
|
+
- - ~>
|
36
|
+
- !ruby/object:Gem::Version
|
37
|
+
version: '3.0'
|
38
|
+
type: :runtime
|
39
|
+
prerelease: false
|
40
|
+
version_requirements: !ruby/object:Gem::Requirement
|
41
|
+
none: false
|
42
|
+
requirements:
|
43
|
+
- - ~>
|
44
|
+
- !ruby/object:Gem::Version
|
45
|
+
version: '3.0'
|
46
|
+
- !ruby/object:Gem::Dependency
|
47
|
+
name: bundler
|
48
|
+
requirement: !ruby/object:Gem::Requirement
|
49
|
+
none: false
|
50
|
+
requirements:
|
51
|
+
- - ~>
|
52
|
+
- !ruby/object:Gem::Version
|
53
|
+
version: 1.1.5
|
54
|
+
type: :development
|
55
|
+
prerelease: false
|
56
|
+
version_requirements: !ruby/object:Gem::Requirement
|
57
|
+
none: false
|
58
|
+
requirements:
|
59
|
+
- - ~>
|
60
|
+
- !ruby/object:Gem::Version
|
61
|
+
version: 1.1.5
|
62
|
+
- !ruby/object:Gem::Dependency
|
63
|
+
name: mocha
|
64
|
+
requirement: !ruby/object:Gem::Requirement
|
65
|
+
none: false
|
66
|
+
requirements:
|
67
|
+
- - ~>
|
68
|
+
- !ruby/object:Gem::Version
|
69
|
+
version: 0.12.1
|
70
|
+
type: :development
|
71
|
+
prerelease: false
|
72
|
+
version_requirements: !ruby/object:Gem::Requirement
|
73
|
+
none: false
|
74
|
+
requirements:
|
75
|
+
- - ~>
|
76
|
+
- !ruby/object:Gem::Version
|
77
|
+
version: 0.12.1
|
78
|
+
- !ruby/object:Gem::Dependency
|
79
|
+
name: rake
|
80
|
+
requirement: !ruby/object:Gem::Requirement
|
81
|
+
none: false
|
82
|
+
requirements:
|
83
|
+
- - ~>
|
84
|
+
- !ruby/object:Gem::Version
|
85
|
+
version: 0.9.2.2
|
86
|
+
type: :development
|
87
|
+
prerelease: false
|
88
|
+
version_requirements: !ruby/object:Gem::Requirement
|
89
|
+
none: false
|
90
|
+
requirements:
|
91
|
+
- - ~>
|
92
|
+
- !ruby/object:Gem::Version
|
93
|
+
version: 0.9.2.2
|
94
|
+
description: Retries reads and transactions when an ActiveRecord::StatementInvalid
|
95
|
+
occurs that matches a list of errors. Default list of errors includes errors related
|
96
|
+
to failover situations to allow for graceful failovers during a request and to attempt
|
97
|
+
prevention of data loss for temporary issue.
|
98
|
+
email:
|
99
|
+
- samuel@kadolph.com
|
100
|
+
executables: []
|
101
|
+
extensions: []
|
102
|
+
extra_rdoc_files: []
|
103
|
+
files:
|
104
|
+
- lib/active_record/retry/rails.rb
|
105
|
+
- lib/active_record/retry/version.rb
|
106
|
+
- lib/active_record/retry.rb
|
107
|
+
- lib/activerecord-retry.rb
|
108
|
+
- test/active_record/retry_test.rb
|
109
|
+
homepage: http://samuelkadolph.github.com/activerecord-retry/
|
110
|
+
licenses: []
|
111
|
+
post_install_message:
|
112
|
+
rdoc_options: []
|
113
|
+
require_paths:
|
114
|
+
- lib
|
115
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
116
|
+
none: false
|
117
|
+
requirements:
|
118
|
+
- - ! '>='
|
119
|
+
- !ruby/object:Gem::Version
|
120
|
+
version: 1.9.2
|
121
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
122
|
+
none: false
|
123
|
+
requirements:
|
124
|
+
- - ! '>='
|
125
|
+
- !ruby/object:Gem::Version
|
126
|
+
version: '0'
|
127
|
+
segments:
|
128
|
+
- 0
|
129
|
+
hash: -2741769345565936512
|
130
|
+
requirements: []
|
131
|
+
rubyforge_project:
|
132
|
+
rubygems_version: 1.8.24
|
133
|
+
signing_key:
|
134
|
+
specification_version: 3
|
135
|
+
summary: activerecord-retry add query retrying to ActiveRecord reads and transactions
|
136
|
+
on specific errors.
|
137
|
+
test_files:
|
138
|
+
- test/active_record/retry_test.rb
|