sidekiq 5.2.1 → 6.0.7
Sign up to get free protection for your applications and to get access to all the features.
Potentially problematic release.
This version of sidekiq might be problematic. Click here for more details.
- checksums.yaml +5 -5
- data/.circleci/config.yml +60 -0
- data/.gitignore +1 -1
- data/.standard.yml +20 -0
- data/6.0-Upgrade.md +72 -0
- data/COMM-LICENSE +11 -9
- data/Changes.md +209 -0
- data/Ent-2.0-Upgrade.md +37 -0
- data/Ent-Changes.md +36 -1
- data/Gemfile +19 -9
- data/Gemfile.lock +208 -0
- data/Pro-5.0-Upgrade.md +25 -0
- data/Pro-Changes.md +44 -1
- data/README.md +19 -31
- data/Rakefile +6 -4
- data/bin/sidekiq +19 -0
- data/bin/sidekiqload +33 -25
- data/bin/sidekiqmon +8 -0
- data/lib/generators/sidekiq/templates/worker_test.rb.erb +1 -1
- data/lib/generators/sidekiq/worker_generator.rb +21 -13
- data/lib/sidekiq/api.rb +240 -214
- data/lib/sidekiq/cli.rb +167 -219
- data/lib/sidekiq/client.rb +61 -46
- data/lib/sidekiq/delay.rb +5 -6
- data/lib/sidekiq/exception_handler.rb +10 -12
- data/lib/sidekiq/extensions/action_mailer.rb +10 -20
- data/lib/sidekiq/extensions/active_record.rb +9 -7
- data/lib/sidekiq/extensions/class_methods.rb +9 -7
- data/lib/sidekiq/extensions/generic_proxy.rb +4 -4
- data/lib/sidekiq/fetch.rb +11 -12
- data/lib/sidekiq/job_logger.rb +47 -9
- data/lib/sidekiq/job_retry.rb +79 -58
- data/lib/sidekiq/launcher.rb +86 -54
- data/lib/sidekiq/logger.rb +165 -0
- data/lib/sidekiq/manager.rb +10 -12
- data/lib/sidekiq/middleware/chain.rb +14 -4
- data/lib/sidekiq/middleware/i18n.rb +5 -7
- data/lib/sidekiq/monitor.rb +133 -0
- data/lib/sidekiq/paginator.rb +18 -14
- data/lib/sidekiq/processor.rb +113 -79
- data/lib/sidekiq/rails.rb +24 -29
- data/lib/sidekiq/redis_connection.rb +42 -24
- data/lib/sidekiq/scheduled.rb +28 -29
- data/lib/sidekiq/sd_notify.rb +149 -0
- data/lib/sidekiq/systemd.rb +24 -0
- data/lib/sidekiq/testing/inline.rb +2 -1
- data/lib/sidekiq/testing.rb +34 -23
- data/lib/sidekiq/util.rb +17 -16
- data/lib/sidekiq/version.rb +2 -1
- data/lib/sidekiq/web/action.rb +14 -10
- data/lib/sidekiq/web/application.rb +79 -69
- data/lib/sidekiq/web/helpers.rb +89 -71
- data/lib/sidekiq/web/router.rb +17 -16
- data/lib/sidekiq/web.rb +41 -49
- data/lib/sidekiq/worker.rb +134 -91
- data/lib/sidekiq.rb +69 -44
- data/sidekiq.gemspec +16 -18
- data/web/assets/javascripts/application.js +22 -19
- data/web/assets/javascripts/dashboard.js +16 -25
- data/web/assets/stylesheets/application-dark.css +122 -0
- data/web/assets/stylesheets/application.css +44 -2
- data/web/assets/stylesheets/bootstrap.css +1 -1
- data/web/locales/ar.yml +1 -0
- data/web/locales/de.yml +14 -2
- data/web/locales/en.yml +3 -0
- data/web/locales/fr.yml +2 -2
- data/web/locales/ja.yml +4 -1
- data/web/locales/lt.yml +83 -0
- data/web/locales/vi.yml +83 -0
- data/web/views/_job_info.erb +2 -1
- data/web/views/_nav.erb +3 -17
- data/web/views/busy.erb +4 -1
- data/web/views/dead.erb +2 -2
- data/web/views/layout.erb +1 -0
- data/web/views/morgue.erb +4 -1
- data/web/views/queue.erb +11 -1
- data/web/views/queues.erb +9 -1
- data/web/views/retries.erb +8 -1
- data/web/views/retry.erb +2 -2
- data/web/views/scheduled.erb +4 -1
- metadata +37 -27
- data/.travis.yml +0 -14
- data/bin/sidekiqctl +0 -99
- data/lib/sidekiq/core_ext.rb +0 -1
- data/lib/sidekiq/logging.rb +0 -122
- data/lib/sidekiq/middleware/server/active_record.rb +0 -23
data/lib/sidekiq/web/action.rb
CHANGED
@@ -2,7 +2,7 @@
|
|
2
2
|
|
3
3
|
module Sidekiq
|
4
4
|
class WebAction
|
5
|
-
RACK_SESSION =
|
5
|
+
RACK_SESSION = "rack.session"
|
6
6
|
|
7
7
|
attr_accessor :env, :block, :type
|
8
8
|
|
@@ -19,14 +19,14 @@ module Sidekiq
|
|
19
19
|
end
|
20
20
|
|
21
21
|
def redirect(location)
|
22
|
-
throw :halt, [302, {
|
22
|
+
throw :halt, [302, {"Location" => "#{request.base_url}#{location}"}, []]
|
23
23
|
end
|
24
24
|
|
25
25
|
def params
|
26
|
-
indifferent_hash = Hash.new {|hash,key| hash[key.to_s] if Symbol === key }
|
26
|
+
indifferent_hash = Hash.new { |hash, key| hash[key.to_s] if Symbol === key }
|
27
27
|
|
28
28
|
indifferent_hash.merge! request.params
|
29
|
-
route_params.each {|k,v| indifferent_hash[k.to_s] = v }
|
29
|
+
route_params.each { |k, v| indifferent_hash[k.to_s] = v }
|
30
30
|
|
31
31
|
indifferent_hash
|
32
32
|
end
|
@@ -40,10 +40,14 @@ module Sidekiq
|
|
40
40
|
end
|
41
41
|
|
42
42
|
def erb(content, options = {})
|
43
|
-
if content.
|
43
|
+
if content.is_a? Symbol
|
44
44
|
unless respond_to?(:"_erb_#{content}")
|
45
45
|
src = ERB.new(File.read("#{Web.settings.views}/#{content}.erb")).src
|
46
|
-
WebAction.class_eval
|
46
|
+
WebAction.class_eval <<-RUBY, __FILE__, __LINE__ + 1
|
47
|
+
def _erb_#{content}
|
48
|
+
#{src}
|
49
|
+
end
|
50
|
+
RUBY
|
47
51
|
end
|
48
52
|
end
|
49
53
|
|
@@ -64,22 +68,22 @@ module Sidekiq
|
|
64
68
|
end
|
65
69
|
|
66
70
|
def json(payload)
|
67
|
-
[200, {
|
71
|
+
[200, {"Content-Type" => "application/json", "Cache-Control" => "no-cache"}, [Sidekiq.dump_json(payload)]]
|
68
72
|
end
|
69
73
|
|
70
74
|
def initialize(env, block)
|
71
75
|
@_erb = false
|
72
76
|
@env = env
|
73
77
|
@block = block
|
74
|
-
|
78
|
+
@files ||= {}
|
75
79
|
end
|
76
80
|
|
77
81
|
private
|
78
82
|
|
79
83
|
def _erb(file, locals)
|
80
|
-
locals
|
84
|
+
locals&.each { |k, v| define_singleton_method(k) { v } unless singleton_methods.include? k }
|
81
85
|
|
82
|
-
if file.
|
86
|
+
if file.is_a?(String)
|
83
87
|
ERB.new(file).result(binding)
|
84
88
|
else
|
85
89
|
send(:"_erb_#{file}")
|
@@ -5,8 +5,7 @@ module Sidekiq
|
|
5
5
|
extend WebRouter
|
6
6
|
|
7
7
|
CONTENT_LENGTH = "Content-Length"
|
8
|
-
|
9
|
-
REDIS_KEYS = %w(redis_version uptime_in_days connected_clients used_memory_human used_memory_peak_human)
|
8
|
+
REDIS_KEYS = %w[redis_version uptime_in_days connected_clients used_memory_human used_memory_peak_human]
|
10
9
|
CSP_HEADER = [
|
11
10
|
"default-src 'self' https: http:",
|
12
11
|
"child-src 'self'",
|
@@ -17,11 +16,11 @@ module Sidekiq
|
|
17
16
|
"manifest-src 'self'",
|
18
17
|
"media-src 'self'",
|
19
18
|
"object-src 'none'",
|
20
|
-
"script-src 'self' https: http:",
|
19
|
+
"script-src 'self' https: http: 'unsafe-inline'",
|
21
20
|
"style-src 'self' https: http: 'unsafe-inline'",
|
22
21
|
"worker-src 'self'",
|
23
22
|
"base-uri 'self'"
|
24
|
-
].join(
|
23
|
+
].join("; ").freeze
|
25
24
|
|
26
25
|
def initialize(klass)
|
27
26
|
@klass = klass
|
@@ -44,8 +43,8 @@ module Sidekiq
|
|
44
43
|
end
|
45
44
|
|
46
45
|
get "/" do
|
47
|
-
@redis_info = redis_info.select{ |k, v| REDIS_KEYS.include? k }
|
48
|
-
stats_history = Sidekiq::Stats::History.new((params[
|
46
|
+
@redis_info = redis_info.select { |k, v| REDIS_KEYS.include? k }
|
47
|
+
stats_history = Sidekiq::Stats::History.new((params["days"] || 30).to_i)
|
49
48
|
@processed_history = stats_history.processed
|
50
49
|
@failed_history = stats_history.failed
|
51
50
|
|
@@ -57,14 +56,14 @@ module Sidekiq
|
|
57
56
|
end
|
58
57
|
|
59
58
|
post "/busy" do
|
60
|
-
if params[
|
61
|
-
p = Sidekiq::Process.new(
|
62
|
-
p.quiet! if params[
|
63
|
-
p.stop! if params[
|
59
|
+
if params["identity"]
|
60
|
+
p = Sidekiq::Process.new("identity" => params["identity"])
|
61
|
+
p.quiet! if params["quiet"]
|
62
|
+
p.stop! if params["stop"]
|
64
63
|
else
|
65
64
|
processes.each do |pro|
|
66
|
-
pro.quiet! if params[
|
67
|
-
pro.stop! if params[
|
65
|
+
pro.quiet! if params["quiet"]
|
66
|
+
pro.stop! if params["stop"]
|
68
67
|
end
|
69
68
|
end
|
70
69
|
|
@@ -82,37 +81,46 @@ module Sidekiq
|
|
82
81
|
|
83
82
|
halt(404) unless @name
|
84
83
|
|
85
|
-
@count = (params[
|
84
|
+
@count = (params["count"] || 25).to_i
|
86
85
|
@queue = Sidekiq::Queue.new(@name)
|
87
|
-
(@current_page, @total_size, @messages) = page("queue:#{@name}", params[
|
86
|
+
(@current_page, @total_size, @messages) = page("queue:#{@name}", params["page"], @count, reverse: params["direction"] == "asc")
|
88
87
|
@messages = @messages.map { |msg| Sidekiq::Job.new(msg, @name) }
|
89
88
|
|
90
89
|
erb(:queue)
|
91
90
|
end
|
92
91
|
|
93
92
|
post "/queues/:name" do
|
94
|
-
Sidekiq::Queue.new(route_params[:name])
|
93
|
+
queue = Sidekiq::Queue.new(route_params[:name])
|
94
|
+
|
95
|
+
if Sidekiq.pro? && params["pause"]
|
96
|
+
queue.pause!
|
97
|
+
elsif Sidekiq.pro? && params["unpause"]
|
98
|
+
queue.unpause!
|
99
|
+
else
|
100
|
+
queue.clear
|
101
|
+
end
|
95
102
|
|
96
103
|
redirect "#{root_path}queues"
|
97
104
|
end
|
98
105
|
|
99
106
|
post "/queues/:name/delete" do
|
100
107
|
name = route_params[:name]
|
101
|
-
Sidekiq::Job.new(params[
|
108
|
+
Sidekiq::Job.new(params["key_val"], name).delete
|
102
109
|
|
103
110
|
redirect_with_query("#{root_path}queues/#{CGI.escape(name)}")
|
104
111
|
end
|
105
112
|
|
106
|
-
get
|
107
|
-
@count = (params[
|
108
|
-
(@current_page, @total_size, @dead) = page("dead", params[
|
113
|
+
get "/morgue" do
|
114
|
+
@count = (params["count"] || 25).to_i
|
115
|
+
(@current_page, @total_size, @dead) = page("dead", params["page"], @count, reverse: true)
|
109
116
|
@dead = @dead.map { |msg, score| Sidekiq::SortedEntry.new(nil, score, msg) }
|
110
117
|
|
111
118
|
erb(:morgue)
|
112
119
|
end
|
113
120
|
|
114
121
|
get "/morgue/:key" do
|
115
|
-
|
122
|
+
key = route_params[:key]
|
123
|
+
halt(404) unless key
|
116
124
|
|
117
125
|
@dead = Sidekiq::DeadSet.new.fetch(*parse_params(key)).first
|
118
126
|
|
@@ -123,10 +131,10 @@ module Sidekiq
|
|
123
131
|
end
|
124
132
|
end
|
125
133
|
|
126
|
-
post
|
127
|
-
redirect(request.path) unless params[
|
134
|
+
post "/morgue" do
|
135
|
+
redirect(request.path) unless params["key"]
|
128
136
|
|
129
|
-
params[
|
137
|
+
params["key"].each do |key|
|
130
138
|
job = Sidekiq::DeadSet.new.fetch(*parse_params(key)).first
|
131
139
|
retry_or_delete_or_kill job, params if job
|
132
140
|
end
|
@@ -147,7 +155,8 @@ module Sidekiq
|
|
147
155
|
end
|
148
156
|
|
149
157
|
post "/morgue/:key" do
|
150
|
-
|
158
|
+
key = route_params[:key]
|
159
|
+
halt(404) unless key
|
151
160
|
|
152
161
|
job = Sidekiq::DeadSet.new.fetch(*parse_params(key)).first
|
153
162
|
retry_or_delete_or_kill job, params if job
|
@@ -155,9 +164,9 @@ module Sidekiq
|
|
155
164
|
redirect_with_query("#{root_path}morgue")
|
156
165
|
end
|
157
166
|
|
158
|
-
get
|
159
|
-
@count = (params[
|
160
|
-
(@current_page, @total_size, @retries) = page("retry", params[
|
167
|
+
get "/retries" do
|
168
|
+
@count = (params["count"] || 25).to_i
|
169
|
+
(@current_page, @total_size, @retries) = page("retry", params["page"], @count)
|
161
170
|
@retries = @retries.map { |msg, score| Sidekiq::SortedEntry.new(nil, score, msg) }
|
162
171
|
|
163
172
|
erb(:retries)
|
@@ -173,10 +182,10 @@ module Sidekiq
|
|
173
182
|
end
|
174
183
|
end
|
175
184
|
|
176
|
-
post
|
177
|
-
redirect(request.path) unless params[
|
185
|
+
post "/retries" do
|
186
|
+
redirect(request.path) unless params["key"]
|
178
187
|
|
179
|
-
params[
|
188
|
+
params["key"].each do |key|
|
180
189
|
job = Sidekiq::RetrySet.new.fetch(*parse_params(key)).first
|
181
190
|
retry_or_delete_or_kill job, params if job
|
182
191
|
end
|
@@ -196,6 +205,12 @@ module Sidekiq
|
|
196
205
|
redirect "#{root_path}retries"
|
197
206
|
end
|
198
207
|
|
208
|
+
post "/retries/all/kill" do
|
209
|
+
Sidekiq::RetrySet.new.kill_all
|
210
|
+
|
211
|
+
redirect "#{root_path}retries"
|
212
|
+
end
|
213
|
+
|
199
214
|
post "/retries/:key" do
|
200
215
|
job = Sidekiq::RetrySet.new.fetch(*parse_params(route_params[:key])).first
|
201
216
|
|
@@ -204,9 +219,9 @@ module Sidekiq
|
|
204
219
|
redirect_with_query("#{root_path}retries")
|
205
220
|
end
|
206
221
|
|
207
|
-
get
|
208
|
-
@count = (params[
|
209
|
-
(@current_page, @total_size, @scheduled) = page("schedule", params[
|
222
|
+
get "/scheduled" do
|
223
|
+
@count = (params["count"] || 25).to_i
|
224
|
+
(@current_page, @total_size, @scheduled) = page("schedule", params["page"], @count)
|
210
225
|
@scheduled = @scheduled.map { |msg, score| Sidekiq::SortedEntry.new(nil, score, msg) }
|
211
226
|
|
212
227
|
erb(:scheduled)
|
@@ -222,10 +237,10 @@ module Sidekiq
|
|
222
237
|
end
|
223
238
|
end
|
224
239
|
|
225
|
-
post
|
226
|
-
redirect(request.path) unless params[
|
240
|
+
post "/scheduled" do
|
241
|
+
redirect(request.path) unless params["key"]
|
227
242
|
|
228
|
-
params[
|
243
|
+
params["key"].each do |key|
|
229
244
|
job = Sidekiq::ScheduledSet.new.fetch(*parse_params(key)).first
|
230
245
|
delete_or_add_queue job, params if job
|
231
246
|
end
|
@@ -234,7 +249,8 @@ module Sidekiq
|
|
234
249
|
end
|
235
250
|
|
236
251
|
post "/scheduled/:key" do
|
237
|
-
|
252
|
+
key = route_params[:key]
|
253
|
+
halt(404) unless key
|
238
254
|
|
239
255
|
job = Sidekiq::ScheduledSet.new.fetch(*parse_params(key)).first
|
240
256
|
delete_or_add_queue job, params if job
|
@@ -242,23 +258,23 @@ module Sidekiq
|
|
242
258
|
redirect_with_query("#{root_path}scheduled")
|
243
259
|
end
|
244
260
|
|
245
|
-
get
|
261
|
+
get "/dashboard/stats" do
|
246
262
|
redirect "#{root_path}stats"
|
247
263
|
end
|
248
264
|
|
249
|
-
get
|
265
|
+
get "/stats" do
|
250
266
|
sidekiq_stats = Sidekiq::Stats.new
|
251
|
-
redis_stats
|
267
|
+
redis_stats = redis_info.select { |k, v| REDIS_KEYS.include? k }
|
252
268
|
json(
|
253
269
|
sidekiq: {
|
254
|
-
processed:
|
255
|
-
failed:
|
256
|
-
busy:
|
257
|
-
processes:
|
258
|
-
enqueued:
|
259
|
-
scheduled:
|
260
|
-
retries:
|
261
|
-
dead:
|
270
|
+
processed: sidekiq_stats.processed,
|
271
|
+
failed: sidekiq_stats.failed,
|
272
|
+
busy: sidekiq_stats.workers_size,
|
273
|
+
processes: sidekiq_stats.processes_size,
|
274
|
+
enqueued: sidekiq_stats.enqueued,
|
275
|
+
scheduled: sidekiq_stats.scheduled_size,
|
276
|
+
retries: sidekiq_stats.retry_size,
|
277
|
+
dead: sidekiq_stats.dead_size,
|
262
278
|
default_latency: sidekiq_stats.default_queue_latency
|
263
279
|
},
|
264
280
|
redis: redis_stats,
|
@@ -266,48 +282,42 @@ module Sidekiq
|
|
266
282
|
)
|
267
283
|
end
|
268
284
|
|
269
|
-
get
|
285
|
+
get "/stats/queues" do
|
270
286
|
json Sidekiq::Stats::Queues.new.lengths
|
271
287
|
end
|
272
288
|
|
273
289
|
def call(env)
|
274
290
|
action = self.class.match(env)
|
275
|
-
return [404, {"Content-Type" => "text/plain", "X-Cascade" => "pass"
|
291
|
+
return [404, {"Content-Type" => "text/plain", "X-Cascade" => "pass"}, ["Not Found"]] unless action
|
276
292
|
|
277
|
-
|
278
|
-
|
293
|
+
app = @klass
|
294
|
+
resp = catch(:halt) do # rubocop:disable Standard/SemanticBlocks
|
279
295
|
self.class.run_befores(app, action)
|
280
|
-
|
281
|
-
|
282
|
-
|
283
|
-
self.class.run_afters(app, action)
|
284
|
-
end
|
285
|
-
|
286
|
-
resp
|
296
|
+
action.instance_exec env, &action.block
|
297
|
+
ensure
|
298
|
+
self.class.run_afters(app, action)
|
287
299
|
end
|
288
300
|
|
289
301
|
resp = case resp
|
290
302
|
when Array
|
303
|
+
# redirects go here
|
291
304
|
resp
|
292
305
|
else
|
306
|
+
# rendered content goes here
|
293
307
|
headers = {
|
294
308
|
"Content-Type" => "text/html",
|
295
309
|
"Cache-Control" => "no-cache",
|
296
310
|
"Content-Language" => action.locale,
|
297
311
|
"Content-Security-Policy" => CSP_HEADER
|
298
312
|
}
|
299
|
-
|
313
|
+
# we'll let Rack calculate Content-Length for us.
|
300
314
|
[200, headers, [resp]]
|
301
315
|
end
|
302
316
|
|
303
|
-
resp[1] = resp[1].dup
|
304
|
-
|
305
|
-
resp[1][CONTENT_LENGTH] = resp[2].inject(0) { |l, p| l + p.bytesize }.to_s
|
306
|
-
|
307
317
|
resp
|
308
318
|
end
|
309
319
|
|
310
|
-
def self.helpers(mod=nil, &block)
|
320
|
+
def self.helpers(mod = nil, &block)
|
311
321
|
if block_given?
|
312
322
|
WebAction.class_eval(&block)
|
313
323
|
else
|
@@ -315,11 +325,11 @@ module Sidekiq
|
|
315
325
|
end
|
316
326
|
end
|
317
327
|
|
318
|
-
def self.before(path=nil, &block)
|
328
|
+
def self.before(path = nil, &block)
|
319
329
|
befores << [path && Regexp.new("\\A#{path.gsub("*", ".*")}\\z"), block]
|
320
330
|
end
|
321
331
|
|
322
|
-
def self.after(path=nil, &block)
|
332
|
+
def self.after(path = nil, &block)
|
323
333
|
afters << [path && Regexp.new("\\A#{path.gsub("*", ".*")}\\z"), block]
|
324
334
|
end
|
325
335
|
|
@@ -332,8 +342,8 @@ module Sidekiq
|
|
332
342
|
end
|
333
343
|
|
334
344
|
def self.run_hooks(hooks, app, action)
|
335
|
-
hooks.select { |p,_| !p || p =~ action.env[WebRouter::PATH_INFO] }
|
336
|
-
|
345
|
+
hooks.select { |p, _| !p || p =~ action.env[WebRouter::PATH_INFO] }
|
346
|
+
.each { |_, b| action.instance_exec(action.env, app, &b) }
|
337
347
|
end
|
338
348
|
|
339
349
|
def self.befores
|