sidekiq 6.5.0 → 6.5.3
Sign up to get free protection for your applications and to get access to all the features.
Potentially problematic release.
This version of sidekiq might be problematic. Click here for more details.
- checksums.yaml +4 -4
- data/Changes.md +22 -2
- data/lib/sidekiq/api.rb +146 -27
- data/lib/sidekiq/cli.rb +2 -1
- data/lib/sidekiq/client.rb +1 -1
- data/lib/sidekiq/component.rb +3 -2
- data/lib/sidekiq/delay.rb +1 -1
- data/lib/sidekiq/fetch.rb +1 -1
- data/lib/sidekiq/job_retry.rb +46 -31
- data/lib/sidekiq/launcher.rb +4 -2
- data/lib/sidekiq/metrics/deploy.rb +47 -0
- data/lib/sidekiq/metrics/query.rb +124 -0
- data/lib/sidekiq/metrics/shared.rb +94 -0
- data/lib/sidekiq/metrics/tracking.rb +134 -0
- data/lib/sidekiq/middleware/chain.rb +70 -35
- data/lib/sidekiq/middleware/current_attributes.rb +6 -4
- data/lib/sidekiq/middleware/modules.rb +2 -0
- data/lib/sidekiq/processor.rb +1 -1
- data/lib/sidekiq/version.rb +1 -1
- data/lib/sidekiq/web/application.rb +13 -0
- data/lib/sidekiq/web/helpers.rb +24 -1
- data/lib/sidekiq/web.rb +4 -0
- data/lib/sidekiq.rb +9 -1
- data/sidekiq.gemspec +1 -1
- data/web/assets/javascripts/application.js +1 -1
- data/web/assets/javascripts/dashboard.js +0 -17
- data/web/assets/javascripts/graph.js +16 -0
- data/web/locales/en.yml +4 -0
- data/web/views/_nav.erb +1 -1
- data/web/views/busy.erb +1 -1
- data/web/views/dashboard.erb +1 -0
- data/web/views/metrics.erb +59 -0
- data/web/views/metrics_for_job.erb +92 -0
- data/web/views/queue.erb +5 -1
- metadata +11 -4
@@ -4,84 +4,98 @@ require "sidekiq/middleware/modules"
|
|
4
4
|
|
5
5
|
module Sidekiq
|
6
6
|
# Middleware is code configured to run before/after
|
7
|
-
# a
|
7
|
+
# a job is processed. It is patterned after Rack
|
8
8
|
# middleware. Middleware exists for the client side
|
9
9
|
# (pushing jobs onto the queue) as well as the server
|
10
10
|
# side (when jobs are actually processed).
|
11
11
|
#
|
12
|
+
# Callers will register middleware Classes and Sidekiq will
|
13
|
+
# create new instances of the middleware for every job. This
|
14
|
+
# is important so that instance state is not shared accidentally
|
15
|
+
# between job executions.
|
16
|
+
#
|
12
17
|
# To add middleware for the client:
|
13
18
|
#
|
14
|
-
#
|
15
|
-
#
|
16
|
-
#
|
19
|
+
# Sidekiq.configure_client do |config|
|
20
|
+
# config.client_middleware do |chain|
|
21
|
+
# chain.add MyClientHook
|
22
|
+
# end
|
17
23
|
# end
|
18
|
-
# end
|
19
24
|
#
|
20
25
|
# To modify middleware for the server, just call
|
21
26
|
# with another block:
|
22
27
|
#
|
23
|
-
#
|
24
|
-
#
|
25
|
-
#
|
26
|
-
#
|
28
|
+
# Sidekiq.configure_server do |config|
|
29
|
+
# config.server_middleware do |chain|
|
30
|
+
# chain.add MyServerHook
|
31
|
+
# chain.remove ActiveRecord
|
32
|
+
# end
|
27
33
|
# end
|
28
|
-
# end
|
29
34
|
#
|
30
35
|
# To insert immediately preceding another entry:
|
31
36
|
#
|
32
|
-
#
|
33
|
-
#
|
34
|
-
#
|
37
|
+
# Sidekiq.configure_client do |config|
|
38
|
+
# config.client_middleware do |chain|
|
39
|
+
# chain.insert_before ActiveRecord, MyClientHook
|
40
|
+
# end
|
35
41
|
# end
|
36
|
-
# end
|
37
42
|
#
|
38
43
|
# To insert immediately after another entry:
|
39
44
|
#
|
40
|
-
#
|
41
|
-
#
|
42
|
-
#
|
45
|
+
# Sidekiq.configure_client do |config|
|
46
|
+
# config.client_middleware do |chain|
|
47
|
+
# chain.insert_after ActiveRecord, MyClientHook
|
48
|
+
# end
|
43
49
|
# end
|
44
|
-
# end
|
45
50
|
#
|
46
51
|
# This is an example of a minimal server middleware:
|
47
52
|
#
|
48
|
-
#
|
49
|
-
#
|
50
|
-
#
|
51
|
-
#
|
52
|
-
#
|
53
|
-
#
|
54
|
-
#
|
53
|
+
# class MyServerHook
|
54
|
+
# include Sidekiq::ServerMiddleware
|
55
|
+
#
|
56
|
+
# def call(job_instance, msg, queue)
|
57
|
+
# logger.info "Before job"
|
58
|
+
# redis {|conn| conn.get("foo") } # do something in Redis
|
59
|
+
# yield
|
60
|
+
# logger.info "After job"
|
61
|
+
# end
|
55
62
|
# end
|
56
|
-
# end
|
57
63
|
#
|
58
64
|
# This is an example of a minimal client middleware, note
|
59
65
|
# the method must return the result or the job will not push
|
60
66
|
# to Redis:
|
61
67
|
#
|
62
|
-
#
|
63
|
-
#
|
64
|
-
#
|
65
|
-
#
|
66
|
-
#
|
67
|
-
#
|
68
|
-
#
|
68
|
+
# class MyClientHook
|
69
|
+
# include Sidekiq::ClientMiddleware
|
70
|
+
#
|
71
|
+
# def call(job_class, msg, queue, redis_pool)
|
72
|
+
# logger.info "Before push"
|
73
|
+
# result = yield
|
74
|
+
# logger.info "After push"
|
75
|
+
# result
|
76
|
+
# end
|
69
77
|
# end
|
70
|
-
# end
|
71
78
|
#
|
72
79
|
module Middleware
|
73
80
|
class Chain
|
74
81
|
include Enumerable
|
75
82
|
|
83
|
+
# A unique instance of the middleware chain is created for
|
84
|
+
# each job executed in order to be thread-safe.
|
85
|
+
# @param copy [Sidekiq::Middleware::Chain] New instance of Chain
|
86
|
+
# @returns nil
|
76
87
|
def initialize_copy(copy)
|
77
88
|
copy.instance_variable_set(:@entries, entries.dup)
|
89
|
+
nil
|
78
90
|
end
|
79
91
|
|
92
|
+
# Iterate through each middleware in the chain
|
80
93
|
def each(&block)
|
81
94
|
entries.each(&block)
|
82
95
|
end
|
83
96
|
|
84
|
-
|
97
|
+
# @api private
|
98
|
+
def initialize(config = nil) # :nodoc:
|
85
99
|
@config = config
|
86
100
|
@entries = nil
|
87
101
|
yield self if block_given?
|
@@ -91,20 +105,33 @@ module Sidekiq
|
|
91
105
|
@entries ||= []
|
92
106
|
end
|
93
107
|
|
108
|
+
# Remove all middleware matching the given Class
|
109
|
+
# @param klass [Class]
|
94
110
|
def remove(klass)
|
95
111
|
entries.delete_if { |entry| entry.klass == klass }
|
96
112
|
end
|
97
113
|
|
114
|
+
# Add the given middleware to the end of the chain.
|
115
|
+
# Sidekiq will call `klass.new(*args)` to create a clean
|
116
|
+
# copy of your middleware for every job executed.
|
117
|
+
#
|
118
|
+
# chain.add(Statsd::Metrics, { collector: "localhost:8125" })
|
119
|
+
#
|
120
|
+
# @param klass [Class] Your middleware class
|
121
|
+
# @param *args [Array<Object>] Set of arguments to pass to every instance of your middleware
|
98
122
|
def add(klass, *args)
|
99
123
|
remove(klass)
|
100
124
|
entries << Entry.new(@config, klass, *args)
|
101
125
|
end
|
102
126
|
|
127
|
+
# Identical to {#add} except the middleware is added to the front of the chain.
|
103
128
|
def prepend(klass, *args)
|
104
129
|
remove(klass)
|
105
130
|
entries.insert(0, Entry.new(@config, klass, *args))
|
106
131
|
end
|
107
132
|
|
133
|
+
# Inserts +newklass+ before +oldklass+ in the chain.
|
134
|
+
# Useful if one middleware must run before another middleware.
|
108
135
|
def insert_before(oldklass, newklass, *args)
|
109
136
|
i = entries.index { |entry| entry.klass == newklass }
|
110
137
|
new_entry = i.nil? ? Entry.new(@config, newklass, *args) : entries.delete_at(i)
|
@@ -112,6 +139,8 @@ module Sidekiq
|
|
112
139
|
entries.insert(i, new_entry)
|
113
140
|
end
|
114
141
|
|
142
|
+
# Inserts +newklass+ after +oldklass+ in the chain.
|
143
|
+
# Useful if one middleware must run after another middleware.
|
115
144
|
def insert_after(oldklass, newklass, *args)
|
116
145
|
i = entries.index { |entry| entry.klass == newklass }
|
117
146
|
new_entry = i.nil? ? Entry.new(@config, newklass, *args) : entries.delete_at(i)
|
@@ -119,10 +148,12 @@ module Sidekiq
|
|
119
148
|
entries.insert(i + 1, new_entry)
|
120
149
|
end
|
121
150
|
|
151
|
+
# @return [Boolean] if the given class is already in the chain
|
122
152
|
def exists?(klass)
|
123
153
|
any? { |entry| entry.klass == klass }
|
124
154
|
end
|
125
155
|
|
156
|
+
# @return [Boolean] if the chain contains no middleware
|
126
157
|
def empty?
|
127
158
|
@entries.nil? || @entries.empty?
|
128
159
|
end
|
@@ -135,6 +166,8 @@ module Sidekiq
|
|
135
166
|
entries.clear
|
136
167
|
end
|
137
168
|
|
169
|
+
# Used by Sidekiq to execute the middleware at runtime
|
170
|
+
# @api private
|
138
171
|
def invoke(*args)
|
139
172
|
return yield if empty?
|
140
173
|
|
@@ -152,6 +185,8 @@ module Sidekiq
|
|
152
185
|
|
153
186
|
private
|
154
187
|
|
188
|
+
# Represents each link in the middleware chain
|
189
|
+
# @api private
|
155
190
|
class Entry
|
156
191
|
attr_reader :klass
|
157
192
|
|
@@ -23,10 +23,12 @@ module Sidekiq
|
|
23
23
|
|
24
24
|
def call(_, job, _, _)
|
25
25
|
attrs = @klass.attributes
|
26
|
-
if
|
27
|
-
job
|
28
|
-
|
29
|
-
|
26
|
+
if attrs.any?
|
27
|
+
if job.has_key?("cattr")
|
28
|
+
job["cattr"].merge!(attrs)
|
29
|
+
else
|
30
|
+
job["cattr"] = attrs
|
31
|
+
end
|
30
32
|
end
|
31
33
|
yield
|
32
34
|
end
|
data/lib/sidekiq/processor.rb
CHANGED
@@ -174,7 +174,7 @@ module Sidekiq
|
|
174
174
|
# signals that we created a retry successfully. We can acknowlege the job.
|
175
175
|
ack = true
|
176
176
|
e = h.cause || h
|
177
|
-
handle_exception(e, {context: "Job raised exception", job: job_hash
|
177
|
+
handle_exception(e, {context: "Job raised exception", job: job_hash})
|
178
178
|
raise e
|
179
179
|
rescue Exception => ex
|
180
180
|
# Unexpected error! This is very bad and indicates an exception that got past
|
data/lib/sidekiq/version.rb
CHANGED
@@ -60,6 +60,19 @@ module Sidekiq
|
|
60
60
|
erb(:dashboard)
|
61
61
|
end
|
62
62
|
|
63
|
+
get "/metrics" do
|
64
|
+
q = Sidekiq::Metrics::Query.new
|
65
|
+
@resultset = q.top_jobs
|
66
|
+
erb(:metrics)
|
67
|
+
end
|
68
|
+
|
69
|
+
get "/metrics/:name" do
|
70
|
+
@name = route_params[:name]
|
71
|
+
q = Sidekiq::Metrics::Query.new
|
72
|
+
@resultset = q.for_job(@name)
|
73
|
+
erb(:metrics_for_job)
|
74
|
+
end
|
75
|
+
|
63
76
|
get "/busy" do
|
64
77
|
erb(:busy)
|
65
78
|
end
|
data/lib/sidekiq/web/helpers.rb
CHANGED
@@ -15,7 +15,7 @@ module Sidekiq
|
|
15
15
|
# so extensions can be localized
|
16
16
|
@strings[lang] ||= settings.locales.each_with_object({}) do |path, global|
|
17
17
|
find_locale_files(lang).each do |file|
|
18
|
-
strs = YAML.
|
18
|
+
strs = YAML.safe_load(File.open(file))
|
19
19
|
global.merge!(strs[lang])
|
20
20
|
end
|
21
21
|
end
|
@@ -148,6 +148,29 @@ module Sidekiq
|
|
148
148
|
@processes ||= Sidekiq::ProcessSet.new
|
149
149
|
end
|
150
150
|
|
151
|
+
# Sorts processes by hostname following the natural sort order so that
|
152
|
+
# 'worker.1' < 'worker.2' < 'worker.10' < 'worker.20'
|
153
|
+
# '2.1.1.1' < '192.168.0.2' < '192.168.0.10'
|
154
|
+
def sorted_processes
|
155
|
+
@sorted_processes ||= begin
|
156
|
+
return processes unless processes.all? { |p| p["hostname"] }
|
157
|
+
|
158
|
+
split_characters = /[._-]/
|
159
|
+
|
160
|
+
padding = processes.flat_map { |p| p["hostname"].split(split_characters) }.map(&:size).max
|
161
|
+
|
162
|
+
processes.to_a.sort_by do |process|
|
163
|
+
process["hostname"].split(split_characters).map do |substring|
|
164
|
+
# Left-pad the substring with '0' if it starts with a number or 'a'
|
165
|
+
# otherwise, so that '25' < 192' < 'a' ('025' < '192' < 'aaa')
|
166
|
+
padding_char = substring[0].match?(/\d/) ? "0" : "a"
|
167
|
+
|
168
|
+
substring.rjust(padding, padding_char)
|
169
|
+
end
|
170
|
+
end
|
171
|
+
end
|
172
|
+
end
|
173
|
+
|
151
174
|
def stats
|
152
175
|
@stats ||= Sidekiq::Stats.new
|
153
176
|
end
|
data/lib/sidekiq/web.rb
CHANGED
data/lib/sidekiq.rb
CHANGED
@@ -34,7 +34,10 @@ module Sidekiq
|
|
34
34
|
startup: [],
|
35
35
|
quiet: [],
|
36
36
|
shutdown: [],
|
37
|
-
heartbeat
|
37
|
+
# triggers when we fire the first heartbeat on startup OR repairing a network partition
|
38
|
+
heartbeat: [],
|
39
|
+
# triggers on EVERY heartbeat call, every 10 seconds
|
40
|
+
beat: []
|
38
41
|
},
|
39
42
|
dead_max_jobs: 10_000,
|
40
43
|
dead_timeout_in_seconds: 180 * 24 * 60 * 60, # 6 months
|
@@ -84,6 +87,11 @@ module Sidekiq
|
|
84
87
|
logger.warn(ex.backtrace.join("\n")) unless ex.backtrace.nil?
|
85
88
|
end
|
86
89
|
|
90
|
+
# DEFAULT_ERROR_HANDLER is a constant that allows the default error handler to
|
91
|
+
# be referenced. It must be defined here, after the default_error_handler
|
92
|
+
# method is defined.
|
93
|
+
DEFAULT_ERROR_HANDLER = method(:default_error_handler)
|
94
|
+
|
87
95
|
@config = DEFAULTS.dup
|
88
96
|
def self.options
|
89
97
|
logger.warn "`config.options[:key] = value` is deprecated, use `config[:key] = value`: #{caller(1..2)}"
|
data/sidekiq.gemspec
CHANGED
@@ -22,7 +22,7 @@ Gem::Specification.new do |gem|
|
|
22
22
|
"source_code_uri" => "https://github.com/mperham/sidekiq"
|
23
23
|
}
|
24
24
|
|
25
|
-
gem.add_dependency "redis", ">= 4.
|
25
|
+
gem.add_dependency "redis", ">= 4.5.0"
|
26
26
|
gem.add_dependency "connection_pool", ">= 2.2.2"
|
27
27
|
gem.add_dependency "rack", "~> 2.0"
|
28
28
|
end
|
@@ -63,7 +63,7 @@ function addPollingListeners(_event) {
|
|
63
63
|
function addDataToggleListeners(event) {
|
64
64
|
var source = event.target || event.srcElement;
|
65
65
|
var targName = source.getAttribute("data-toggle");
|
66
|
-
var full = document.getElementById(targName
|
66
|
+
var full = document.getElementById(targName);
|
67
67
|
if (full.style.display == "block") {
|
68
68
|
full.style.display = 'none';
|
69
69
|
} else {
|