sidekiq-failures 0.0.3 → 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/CHANGELOG.md +7 -0
- data/README.md +66 -9
- data/lib/sidekiq/failures.rb +26 -3
- data/lib/sidekiq/failures/middleware.rb +49 -1
- data/lib/sidekiq/failures/version.rb +1 -1
- data/lib/sidekiq/failures/views/failures.slim +6 -3
- data/lib/sidekiq/failures/web_extension.rb +6 -1
- data/test/middleware_test.rb +110 -7
- data/test/test_helper.rb +8 -0
- data/test/web_extension_test.rb +32 -2
- metadata +4 -4
data/CHANGELOG.md
CHANGED
@@ -1,4 +1,11 @@
|
|
1
1
|
## Unreleased
|
2
|
+
* Allow per worker configuration of failure tracking mode. Thanks to
|
3
|
+
@kbaum for most of the work.
|
4
|
+
* Prevent sidekiq-failures from loading up sidekiq/processor (and thus
|
5
|
+
Celluloid actors) except for inside a Sidekiq server context (@cheald)
|
6
|
+
* Fix pagination bug
|
7
|
+
* Add failures default mode option (@kbaum)
|
8
|
+
* Add checkbox option to reset failed counter (@krasnoukhov)
|
2
9
|
|
3
10
|
## 0.0.3
|
4
11
|
|
data/README.md
CHANGED
@@ -5,8 +5,9 @@ them. Makes use of Sidekiq's custom tabs and middleware chain.
|
|
5
5
|
|
6
6
|
It mimics the way Resque keeps track of failures.
|
7
7
|
|
8
|
-
TIP: Note that each failed job/retry
|
9
|
-
only be removed by you manually. This might result in a pretty big failures list
|
8
|
+
TIP: Note that each failed job/retry might create a new failed job that will
|
9
|
+
only be removed by you manually. This might result in a pretty big failures list
|
10
|
+
depending on how you configure failures tracking in your workers.
|
10
11
|
|
11
12
|
## Installation
|
12
13
|
|
@@ -20,17 +21,73 @@ gem 'sidekiq-failures'
|
|
20
21
|
|
21
22
|
Depends on Sidekiq >= 2.2.1
|
22
23
|
|
23
|
-
## Usage
|
24
|
+
## Usage and Modes
|
24
25
|
|
25
|
-
Simply having the gem in your Gemfile
|
26
|
+
Simply having the gem in your Gemfile is enough to get you started. Your failed jobs will be visible via a Failures tab in the Web UI.
|
26
27
|
|
27
|
-
|
28
|
+
Sidekiq-failures offers three failures tracking options (per worker):
|
28
29
|
|
29
|
-
|
30
|
+
### all (default)
|
30
31
|
|
31
|
-
|
32
|
-
|
33
|
-
|
32
|
+
Tracks failures everytime a background job fails. This mean a job with 25 retries enabled might generate up to 25 failure entries. If the worker has retry disabled only one failure will be tracked.
|
33
|
+
|
34
|
+
This is the default behavior but can be made explicit with:
|
35
|
+
|
36
|
+
```ruby
|
37
|
+
class MyWorker
|
38
|
+
include Sidekiq::Worker
|
39
|
+
|
40
|
+
sidekiq_options :failures => true # or :all
|
41
|
+
|
42
|
+
def perform; end
|
43
|
+
end
|
44
|
+
```
|
45
|
+
|
46
|
+
### exhausted
|
47
|
+
|
48
|
+
Only track failures if the job exhausts all its retries (or doesn't have retries enabled).
|
49
|
+
|
50
|
+
You can set this mode as follows:
|
51
|
+
|
52
|
+
```ruby
|
53
|
+
class MyWorker
|
54
|
+
include Sidekiq::Worker
|
55
|
+
|
56
|
+
sidekiq_options :failures => :exhausted
|
57
|
+
|
58
|
+
def perform; end
|
59
|
+
end
|
60
|
+
```
|
61
|
+
|
62
|
+
### off
|
63
|
+
|
64
|
+
You can also completely turn off failures tracking for a given worker as follows:
|
65
|
+
|
66
|
+
```ruby
|
67
|
+
class MyWorker
|
68
|
+
include Sidekiq::Worker
|
69
|
+
|
70
|
+
sidekiq_options :failures => false # or :off
|
71
|
+
|
72
|
+
def perform; end
|
73
|
+
end
|
74
|
+
```
|
75
|
+
|
76
|
+
### Change the default mode
|
77
|
+
|
78
|
+
You can also change the default of all your workers at once by setting the following server config:
|
79
|
+
|
80
|
+
```ruby
|
81
|
+
Sidekiq.configure_server do |config|
|
82
|
+
config.failures_default_mode = :off
|
83
|
+
end
|
84
|
+
```
|
85
|
+
|
86
|
+
The valid modes are `:all`, `:exhausted` or `:off`.
|
87
|
+
|
88
|
+
## TODO
|
89
|
+
|
90
|
+
* Allow triggering retry of specific failed jobs via Web UI.
|
34
91
|
|
35
92
|
## Contributing
|
36
93
|
|
data/lib/sidekiq/failures.rb
CHANGED
@@ -1,10 +1,31 @@
|
|
1
1
|
require "sidekiq/web"
|
2
|
-
require "sidekiq/processor"
|
3
2
|
require "sidekiq/failures/version"
|
4
3
|
require "sidekiq/failures/middleware"
|
5
4
|
require "sidekiq/failures/web_extension"
|
6
5
|
|
7
6
|
module Sidekiq
|
7
|
+
|
8
|
+
SIDEKIQ_FAILURES_MODES = [:all, :exhausted, :off].freeze
|
9
|
+
|
10
|
+
# Sets the default failure tracking mode.
|
11
|
+
#
|
12
|
+
# The value provided here will be the default behavior but can be overwritten
|
13
|
+
# per worker by using `sidekiq_options :failures => :mode`
|
14
|
+
#
|
15
|
+
# Defaults to :all
|
16
|
+
def self.failures_default_mode=(mode)
|
17
|
+
unless SIDEKIQ_FAILURES_MODES.include?(mode.to_sym)
|
18
|
+
raise ArgumentError, "Sidekiq#failures_default_mode valid options: #{SIDEKIQ_FAILURES_MODES}"
|
19
|
+
end
|
20
|
+
|
21
|
+
@failures_default_mode = mode.to_sym
|
22
|
+
end
|
23
|
+
|
24
|
+
# Fetches the default failure tracking mode.
|
25
|
+
def self.failures_default_mode
|
26
|
+
@failures_default_mode || :all
|
27
|
+
end
|
28
|
+
|
8
29
|
module Failures
|
9
30
|
end
|
10
31
|
end
|
@@ -18,6 +39,8 @@ else
|
|
18
39
|
Sidekiq::Web.tabs["Failures"] = "failures"
|
19
40
|
end
|
20
41
|
|
21
|
-
Sidekiq.
|
22
|
-
|
42
|
+
Sidekiq.configure_server do |config|
|
43
|
+
config.server_middleware do |chain|
|
44
|
+
chain.add Sidekiq::Failures::Middleware
|
45
|
+
end
|
23
46
|
end
|
@@ -1,9 +1,14 @@
|
|
1
1
|
module Sidekiq
|
2
2
|
module Failures
|
3
3
|
class Middleware
|
4
|
+
attr_accessor :msg
|
5
|
+
|
4
6
|
def call(worker, msg, queue)
|
7
|
+
self.msg = msg
|
5
8
|
yield
|
6
9
|
rescue => e
|
10
|
+
raise e if skip_failure?
|
11
|
+
|
7
12
|
data = {
|
8
13
|
:failed_at => Time.now.strftime("%Y/%m/%d %H:%M:%S %Z"),
|
9
14
|
:payload => msg,
|
@@ -16,7 +21,50 @@ module Sidekiq
|
|
16
21
|
|
17
22
|
Sidekiq.redis { |conn| conn.lpush(:failed, Sidekiq.dump_json(data)) }
|
18
23
|
|
19
|
-
raise
|
24
|
+
raise e
|
25
|
+
end
|
26
|
+
|
27
|
+
private
|
28
|
+
|
29
|
+
def skip_failure?
|
30
|
+
failure_mode == :off || not_exhausted?
|
31
|
+
end
|
32
|
+
|
33
|
+
def not_exhausted?
|
34
|
+
failure_mode == :exhausted && !last_try?
|
35
|
+
end
|
36
|
+
|
37
|
+
def failure_mode
|
38
|
+
case msg['failures'].to_s
|
39
|
+
when 'true', 'all'
|
40
|
+
:all
|
41
|
+
when 'false', 'off'
|
42
|
+
:off
|
43
|
+
when 'exhausted'
|
44
|
+
:exhausted
|
45
|
+
else
|
46
|
+
Sidekiq.failures_default_mode
|
47
|
+
end
|
48
|
+
end
|
49
|
+
|
50
|
+
def last_try?
|
51
|
+
retry_count == max_retries - 1
|
52
|
+
end
|
53
|
+
|
54
|
+
def retry_count
|
55
|
+
msg['retry_count'] || 0
|
56
|
+
end
|
57
|
+
|
58
|
+
def max_retries
|
59
|
+
retry_middleware.retry_attempts_from(msg['retry'], default_max_retries)
|
60
|
+
end
|
61
|
+
|
62
|
+
def retry_middleware
|
63
|
+
@retry_middleware ||= Sidekiq::Middleware::Server::RetryJobs.new
|
64
|
+
end
|
65
|
+
|
66
|
+
def default_max_retries
|
67
|
+
Sidekiq::Middleware::Server::RetryJobs::DEFAULT_MAX_RETRY_ATTEMPTS
|
20
68
|
end
|
21
69
|
end
|
22
70
|
end
|
@@ -3,7 +3,7 @@ header.row
|
|
3
3
|
h3 Failed Jobs
|
4
4
|
.span4
|
5
5
|
- if @messages.size > 0
|
6
|
-
== slim :_paging, :locals => { :url => "#{root_path}failures
|
6
|
+
== slim :_paging, :locals => { :url => "#{root_path}failures#@name" }
|
7
7
|
|
8
8
|
- if @messages.size > 0
|
9
9
|
table class="table table-striped table-bordered table-white" style="width: 100%; margin: 0; table-layout:fixed;"
|
@@ -28,9 +28,12 @@ header.row
|
|
28
28
|
|
29
29
|
div.row
|
30
30
|
.span5
|
31
|
-
form.form-
|
31
|
+
form.form-inline action="#{root_path}failures/remove" method="post" style="margin: 20px 0"
|
32
32
|
input.btn.btn-danger.btn-small type="submit" name="delete" value="Clear All"
|
33
|
+
label class="checkbox"
|
34
|
+
input type="checkbox" name="counter" value="true"
|
35
|
+
= "reset failed counter"
|
33
36
|
.span4
|
34
|
-
== slim :_paging, :locals => { :url => "#{root_path}failures
|
37
|
+
== slim :_paging, :locals => { :url => "#{root_path}failures#@name" }
|
35
38
|
- else
|
36
39
|
.alert.alert-success No failed jobs found.
|
@@ -20,7 +20,12 @@ module Sidekiq
|
|
20
20
|
end
|
21
21
|
|
22
22
|
app.post "/failures/remove" do
|
23
|
-
Sidekiq.redis {|c|
|
23
|
+
Sidekiq.redis {|c|
|
24
|
+
c.multi do
|
25
|
+
c.del("failed")
|
26
|
+
c.set("stat:failed", 0) if params["counter"]
|
27
|
+
end
|
28
|
+
}
|
24
29
|
|
25
30
|
redirect "#{root_path}failures"
|
26
31
|
end
|
data/test/middleware_test.rb
CHANGED
@@ -3,19 +3,20 @@ require "test_helper"
|
|
3
3
|
module Sidekiq
|
4
4
|
module Failures
|
5
5
|
describe "Middleware" do
|
6
|
-
TestException = Class.new(StandardError)
|
7
|
-
|
8
6
|
before do
|
9
7
|
$invokes = 0
|
10
8
|
boss = MiniTest::Mock.new
|
11
9
|
@processor = ::Sidekiq::Processor.new(boss)
|
10
|
+
Sidekiq.server_middleware {|chain| chain.add Sidekiq::Failures::Middleware }
|
12
11
|
Sidekiq.redis = REDIS
|
13
12
|
Sidekiq.redis { |c| c.flushdb }
|
13
|
+
Sidekiq.instance_eval { @failures_default_mode = nil }
|
14
14
|
end
|
15
15
|
|
16
|
+
TestException = Class.new(StandardError)
|
17
|
+
|
16
18
|
class MockWorker
|
17
19
|
include Sidekiq::Worker
|
18
|
-
sidekiq_options :retry => false
|
19
20
|
|
20
21
|
def perform(args)
|
21
22
|
$invokes += 1
|
@@ -23,19 +24,121 @@ module Sidekiq
|
|
23
24
|
end
|
24
25
|
end
|
25
26
|
|
26
|
-
it '
|
27
|
-
|
27
|
+
it 'raises an error when failures_default_mode is configured incorrectly' do
|
28
|
+
assert_raises ArgumentError do
|
29
|
+
Sidekiq.failures_default_mode = 'exhaustion'
|
30
|
+
end
|
31
|
+
end
|
32
|
+
|
33
|
+
it 'defaults failures_default_mode to all' do
|
34
|
+
assert_equal :all, Sidekiq.failures_default_mode
|
35
|
+
end
|
36
|
+
|
37
|
+
it 'records all failures by default' do
|
38
|
+
msg = create_message('class' => MockWorker.to_s, 'args' => ['myarg'])
|
39
|
+
|
40
|
+
assert_equal 0, failures_count
|
41
|
+
|
42
|
+
assert_raises TestException do
|
43
|
+
@processor.process(msg, 'default')
|
44
|
+
end
|
45
|
+
|
46
|
+
assert_equal 1, failures_count
|
47
|
+
assert_equal 1, $invokes
|
48
|
+
end
|
49
|
+
|
50
|
+
it 'records all failures if explicitly told to' do
|
51
|
+
msg = create_message('class' => MockWorker.to_s, 'args' => ['myarg'], 'failures' => true)
|
52
|
+
|
53
|
+
assert_equal 0, failures_count
|
54
|
+
|
55
|
+
assert_raises TestException do
|
56
|
+
@processor.process(msg, 'default')
|
57
|
+
end
|
58
|
+
|
59
|
+
assert_equal 1, failures_count
|
60
|
+
assert_equal 1, $invokes
|
61
|
+
end
|
62
|
+
|
63
|
+
it "doesn't record failure if failures disabled" do
|
64
|
+
msg = create_message('class' => MockWorker.to_s, 'args' => ['myarg'], 'failures' => false)
|
28
65
|
|
29
|
-
|
66
|
+
assert_equal 0, failures_count
|
30
67
|
|
31
68
|
assert_raises TestException do
|
32
69
|
@processor.process(msg, 'default')
|
33
70
|
end
|
34
71
|
|
35
|
-
|
72
|
+
assert_equal 0, failures_count
|
73
|
+
assert_equal 1, $invokes
|
74
|
+
end
|
75
|
+
|
76
|
+
it "doesn't record failure if going to be retired again and configured to track exhaustion by default" do
|
77
|
+
Sidekiq.failures_default_mode = :exhausted
|
78
|
+
|
79
|
+
msg = create_message('class' => MockWorker.to_s, 'args' => ['myarg'] )
|
80
|
+
|
81
|
+
assert_equal 0, failures_count
|
82
|
+
|
83
|
+
assert_raises TestException do
|
84
|
+
@processor.process(msg, 'default')
|
85
|
+
end
|
36
86
|
|
87
|
+
assert_equal 0, failures_count
|
37
88
|
assert_equal 1, $invokes
|
38
89
|
end
|
90
|
+
|
91
|
+
|
92
|
+
it "doesn't record failure if going to be retired again and configured to track exhaustion" do
|
93
|
+
msg = create_message('class' => MockWorker.to_s, 'args' => ['myarg'], 'failures' => 'exhausted')
|
94
|
+
|
95
|
+
assert_equal 0, failures_count
|
96
|
+
|
97
|
+
assert_raises TestException do
|
98
|
+
@processor.process(msg, 'default')
|
99
|
+
end
|
100
|
+
|
101
|
+
assert_equal 0, failures_count
|
102
|
+
assert_equal 1, $invokes
|
103
|
+
end
|
104
|
+
|
105
|
+
it "records failure if failing last retry and configured to track exhaustion" do
|
106
|
+
msg = create_message('class' => MockWorker.to_s, 'args' => ['myarg'], 'retry_count' => 24, 'failures' => 'exhausted')
|
107
|
+
|
108
|
+
assert_equal 0, failures_count
|
109
|
+
|
110
|
+
assert_raises TestException do
|
111
|
+
@processor.process(msg, 'default')
|
112
|
+
end
|
113
|
+
|
114
|
+
assert_equal 1, failures_count
|
115
|
+
assert_equal 1, $invokes
|
116
|
+
end
|
117
|
+
|
118
|
+
it "records failure if failing last retry and configured to track exhaustion by default" do
|
119
|
+
Sidekiq.failures_default_mode = 'exhausted'
|
120
|
+
|
121
|
+
msg = create_message('class' => MockWorker.to_s, 'args' => ['myarg'], 'retry_count' => 24)
|
122
|
+
|
123
|
+
assert_equal 0, failures_count
|
124
|
+
|
125
|
+
assert_raises TestException do
|
126
|
+
@processor.process(msg, 'default')
|
127
|
+
end
|
128
|
+
|
129
|
+
assert_equal 1, failures_count
|
130
|
+
assert_equal 1, $invokes
|
131
|
+
end
|
132
|
+
|
133
|
+
|
134
|
+
|
135
|
+
def failures_count
|
136
|
+
Sidekiq.redis { |conn|conn.llen('failed') } || 0
|
137
|
+
end
|
138
|
+
|
139
|
+
def create_message(params)
|
140
|
+
Sidekiq.dump_json(params)
|
141
|
+
end
|
39
142
|
end
|
40
143
|
end
|
41
144
|
end
|
data/test/test_helper.rb
CHANGED
@@ -5,10 +5,18 @@ require "minitest/autorun"
|
|
5
5
|
require "minitest/spec"
|
6
6
|
require "minitest/mock"
|
7
7
|
|
8
|
+
# FIXME Remove once https://github.com/mperham/sidekiq/pull/548 is released.
|
9
|
+
class String
|
10
|
+
def blank?
|
11
|
+
self !~ /[^[:space:]]/
|
12
|
+
end
|
13
|
+
end
|
14
|
+
|
8
15
|
require "rack/test"
|
9
16
|
|
10
17
|
require "sidekiq"
|
11
18
|
require "sidekiq-failures"
|
19
|
+
require "sidekiq/processor"
|
12
20
|
|
13
21
|
Celluloid.logger = nil
|
14
22
|
Sidekiq.logger.level = Logger::ERROR
|
data/test/web_extension_test.rb
CHANGED
@@ -49,9 +49,12 @@ module Sidekiq
|
|
49
49
|
last_response.body.must_match /failures\/remove/
|
50
50
|
last_response.body.must_match /method=\"post/
|
51
51
|
last_response.body.must_match /Clear All/
|
52
|
+
last_response.body.must_match /reset failed counter/
|
52
53
|
end
|
53
54
|
|
54
|
-
it 'can remove all failures' do
|
55
|
+
it 'can remove all failures without clearing counter' do
|
56
|
+
assert_equal failed_count, "1"
|
57
|
+
|
55
58
|
last_response.body.must_match /HardWorker/
|
56
59
|
|
57
60
|
post '/failures/remove'
|
@@ -61,6 +64,24 @@ module Sidekiq
|
|
61
64
|
get '/failures'
|
62
65
|
last_response.status.must_equal 200
|
63
66
|
last_response.body.must_match /No failed jobs found/
|
67
|
+
|
68
|
+
assert_equal failed_count, "1"
|
69
|
+
end
|
70
|
+
|
71
|
+
it 'can remove all failures and clear counter' do
|
72
|
+
assert_equal failed_count, "1"
|
73
|
+
|
74
|
+
last_response.body.must_match /HardWorker/
|
75
|
+
|
76
|
+
post '/failures/remove', counter: "true"
|
77
|
+
last_response.status.must_equal 302
|
78
|
+
last_response.location.must_match /failures$/
|
79
|
+
|
80
|
+
get '/failures'
|
81
|
+
last_response.status.must_equal 200
|
82
|
+
last_response.body.must_match /No failed jobs found/
|
83
|
+
|
84
|
+
assert_equal failed_count, "0"
|
64
85
|
end
|
65
86
|
end
|
66
87
|
|
@@ -75,7 +96,16 @@ module Sidekiq
|
|
75
96
|
:queue => 'default'
|
76
97
|
}
|
77
98
|
|
78
|
-
Sidekiq.redis
|
99
|
+
Sidekiq.redis do |c|
|
100
|
+
c.multi do
|
101
|
+
c.rpush("failed", Sidekiq.dump_json(data))
|
102
|
+
c.set("stat:failed", 1)
|
103
|
+
end
|
104
|
+
end
|
105
|
+
end
|
106
|
+
|
107
|
+
def failed_count
|
108
|
+
Sidekiq.redis { |c| c.get("stat:failed") }
|
79
109
|
end
|
80
110
|
end
|
81
111
|
end
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: sidekiq-failures
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.0
|
4
|
+
version: 0.1.0
|
5
5
|
prerelease:
|
6
6
|
platform: ruby
|
7
7
|
authors:
|
@@ -9,7 +9,7 @@ authors:
|
|
9
9
|
autorequire:
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
|
-
date: 2012-
|
12
|
+
date: 2012-12-30 00:00:00.000000000 Z
|
13
13
|
dependencies:
|
14
14
|
- !ruby/object:Gem::Dependency
|
15
15
|
name: sidekiq
|
@@ -144,7 +144,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
144
144
|
version: '0'
|
145
145
|
segments:
|
146
146
|
- 0
|
147
|
-
hash: -
|
147
|
+
hash: -4591242293469057094
|
148
148
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
149
149
|
none: false
|
150
150
|
requirements:
|
@@ -153,7 +153,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
153
153
|
version: '0'
|
154
154
|
segments:
|
155
155
|
- 0
|
156
|
-
hash: -
|
156
|
+
hash: -4591242293469057094
|
157
157
|
requirements: []
|
158
158
|
rubyforge_project:
|
159
159
|
rubygems_version: 1.8.23
|