sidekiq 5.2.1 → 6.0.7

Sign up to get free protection for your applications and to get access to all the features.

Potentially problematic release.


This version of sidekiq might be problematic. Click here for more details.

Files changed (86) hide show
  1. checksums.yaml +5 -5
  2. data/.circleci/config.yml +60 -0
  3. data/.gitignore +1 -1
  4. data/.standard.yml +20 -0
  5. data/6.0-Upgrade.md +72 -0
  6. data/COMM-LICENSE +11 -9
  7. data/Changes.md +209 -0
  8. data/Ent-2.0-Upgrade.md +37 -0
  9. data/Ent-Changes.md +36 -1
  10. data/Gemfile +19 -9
  11. data/Gemfile.lock +208 -0
  12. data/Pro-5.0-Upgrade.md +25 -0
  13. data/Pro-Changes.md +44 -1
  14. data/README.md +19 -31
  15. data/Rakefile +6 -4
  16. data/bin/sidekiq +19 -0
  17. data/bin/sidekiqload +33 -25
  18. data/bin/sidekiqmon +8 -0
  19. data/lib/generators/sidekiq/templates/worker_test.rb.erb +1 -1
  20. data/lib/generators/sidekiq/worker_generator.rb +21 -13
  21. data/lib/sidekiq/api.rb +240 -214
  22. data/lib/sidekiq/cli.rb +167 -219
  23. data/lib/sidekiq/client.rb +61 -46
  24. data/lib/sidekiq/delay.rb +5 -6
  25. data/lib/sidekiq/exception_handler.rb +10 -12
  26. data/lib/sidekiq/extensions/action_mailer.rb +10 -20
  27. data/lib/sidekiq/extensions/active_record.rb +9 -7
  28. data/lib/sidekiq/extensions/class_methods.rb +9 -7
  29. data/lib/sidekiq/extensions/generic_proxy.rb +4 -4
  30. data/lib/sidekiq/fetch.rb +11 -12
  31. data/lib/sidekiq/job_logger.rb +47 -9
  32. data/lib/sidekiq/job_retry.rb +79 -58
  33. data/lib/sidekiq/launcher.rb +86 -54
  34. data/lib/sidekiq/logger.rb +165 -0
  35. data/lib/sidekiq/manager.rb +10 -12
  36. data/lib/sidekiq/middleware/chain.rb +14 -4
  37. data/lib/sidekiq/middleware/i18n.rb +5 -7
  38. data/lib/sidekiq/monitor.rb +133 -0
  39. data/lib/sidekiq/paginator.rb +18 -14
  40. data/lib/sidekiq/processor.rb +113 -79
  41. data/lib/sidekiq/rails.rb +24 -29
  42. data/lib/sidekiq/redis_connection.rb +42 -24
  43. data/lib/sidekiq/scheduled.rb +28 -29
  44. data/lib/sidekiq/sd_notify.rb +149 -0
  45. data/lib/sidekiq/systemd.rb +24 -0
  46. data/lib/sidekiq/testing/inline.rb +2 -1
  47. data/lib/sidekiq/testing.rb +34 -23
  48. data/lib/sidekiq/util.rb +17 -16
  49. data/lib/sidekiq/version.rb +2 -1
  50. data/lib/sidekiq/web/action.rb +14 -10
  51. data/lib/sidekiq/web/application.rb +79 -69
  52. data/lib/sidekiq/web/helpers.rb +89 -71
  53. data/lib/sidekiq/web/router.rb +17 -16
  54. data/lib/sidekiq/web.rb +41 -49
  55. data/lib/sidekiq/worker.rb +134 -91
  56. data/lib/sidekiq.rb +69 -44
  57. data/sidekiq.gemspec +16 -18
  58. data/web/assets/javascripts/application.js +22 -19
  59. data/web/assets/javascripts/dashboard.js +16 -25
  60. data/web/assets/stylesheets/application-dark.css +122 -0
  61. data/web/assets/stylesheets/application.css +44 -2
  62. data/web/assets/stylesheets/bootstrap.css +1 -1
  63. data/web/locales/ar.yml +1 -0
  64. data/web/locales/de.yml +14 -2
  65. data/web/locales/en.yml +3 -0
  66. data/web/locales/fr.yml +2 -2
  67. data/web/locales/ja.yml +4 -1
  68. data/web/locales/lt.yml +83 -0
  69. data/web/locales/vi.yml +83 -0
  70. data/web/views/_job_info.erb +2 -1
  71. data/web/views/_nav.erb +3 -17
  72. data/web/views/busy.erb +4 -1
  73. data/web/views/dead.erb +2 -2
  74. data/web/views/layout.erb +1 -0
  75. data/web/views/morgue.erb +4 -1
  76. data/web/views/queue.erb +11 -1
  77. data/web/views/queues.erb +9 -1
  78. data/web/views/retries.erb +8 -1
  79. data/web/views/retry.erb +2 -2
  80. data/web/views/scheduled.erb +4 -1
  81. metadata +37 -27
  82. data/.travis.yml +0 -14
  83. data/bin/sidekiqctl +0 -99
  84. data/lib/sidekiq/core_ext.rb +0 -1
  85. data/lib/sidekiq/logging.rb +0 -122
  86. data/lib/sidekiq/middleware/server/active_record.rb +0 -23
@@ -2,7 +2,7 @@
2
2
 
3
3
  module Sidekiq
4
4
  class WebAction
5
- RACK_SESSION = 'rack.session'
5
+ RACK_SESSION = "rack.session"
6
6
 
7
7
  attr_accessor :env, :block, :type
8
8
 
@@ -19,14 +19,14 @@ module Sidekiq
19
19
  end
20
20
 
21
21
  def redirect(location)
22
- throw :halt, [302, { "Location" => "#{request.base_url}#{location}" }, []]
22
+ throw :halt, [302, {"Location" => "#{request.base_url}#{location}"}, []]
23
23
  end
24
24
 
25
25
  def params
26
- indifferent_hash = Hash.new {|hash,key| hash[key.to_s] if Symbol === key }
26
+ indifferent_hash = Hash.new { |hash, key| hash[key.to_s] if Symbol === key }
27
27
 
28
28
  indifferent_hash.merge! request.params
29
- route_params.each {|k,v| indifferent_hash[k.to_s] = v }
29
+ route_params.each { |k, v| indifferent_hash[k.to_s] = v }
30
30
 
31
31
  indifferent_hash
32
32
  end
@@ -40,10 +40,14 @@ module Sidekiq
40
40
  end
41
41
 
42
42
  def erb(content, options = {})
43
- if content.kind_of? Symbol
43
+ if content.is_a? Symbol
44
44
  unless respond_to?(:"_erb_#{content}")
45
45
  src = ERB.new(File.read("#{Web.settings.views}/#{content}.erb")).src
46
- WebAction.class_eval("def _erb_#{content}\n#{src}\n end")
46
+ WebAction.class_eval <<-RUBY, __FILE__, __LINE__ + 1
47
+ def _erb_#{content}
48
+ #{src}
49
+ end
50
+ RUBY
47
51
  end
48
52
  end
49
53
 
@@ -64,22 +68,22 @@ module Sidekiq
64
68
  end
65
69
 
66
70
  def json(payload)
67
- [200, { "Content-Type" => "application/json", "Cache-Control" => "no-cache" }, [Sidekiq.dump_json(payload)]]
71
+ [200, {"Content-Type" => "application/json", "Cache-Control" => "no-cache"}, [Sidekiq.dump_json(payload)]]
68
72
  end
69
73
 
70
74
  def initialize(env, block)
71
75
  @_erb = false
72
76
  @env = env
73
77
  @block = block
74
- @@files ||= {}
78
+ @files ||= {}
75
79
  end
76
80
 
77
81
  private
78
82
 
79
83
  def _erb(file, locals)
80
- locals.each {|k, v| define_singleton_method(k){ v } unless (singleton_methods.include? k)} if locals
84
+ locals&.each { |k, v| define_singleton_method(k) { v } unless singleton_methods.include? k }
81
85
 
82
- if file.kind_of?(String)
86
+ if file.is_a?(String)
83
87
  ERB.new(file).result(binding)
84
88
  else
85
89
  send(:"_erb_#{file}")
@@ -5,8 +5,7 @@ module Sidekiq
5
5
  extend WebRouter
6
6
 
7
7
  CONTENT_LENGTH = "Content-Length"
8
- CONTENT_TYPE = "Content-Type"
9
- REDIS_KEYS = %w(redis_version uptime_in_days connected_clients used_memory_human used_memory_peak_human)
8
+ REDIS_KEYS = %w[redis_version uptime_in_days connected_clients used_memory_human used_memory_peak_human]
10
9
  CSP_HEADER = [
11
10
  "default-src 'self' https: http:",
12
11
  "child-src 'self'",
@@ -17,11 +16,11 @@ module Sidekiq
17
16
  "manifest-src 'self'",
18
17
  "media-src 'self'",
19
18
  "object-src 'none'",
20
- "script-src 'self' https: http:",
19
+ "script-src 'self' https: http: 'unsafe-inline'",
21
20
  "style-src 'self' https: http: 'unsafe-inline'",
22
21
  "worker-src 'self'",
23
22
  "base-uri 'self'"
24
- ].join('; ').freeze
23
+ ].join("; ").freeze
25
24
 
26
25
  def initialize(klass)
27
26
  @klass = klass
@@ -44,8 +43,8 @@ module Sidekiq
44
43
  end
45
44
 
46
45
  get "/" do
47
- @redis_info = redis_info.select{ |k, v| REDIS_KEYS.include? k }
48
- stats_history = Sidekiq::Stats::History.new((params['days'] || 30).to_i)
46
+ @redis_info = redis_info.select { |k, v| REDIS_KEYS.include? k }
47
+ stats_history = Sidekiq::Stats::History.new((params["days"] || 30).to_i)
49
48
  @processed_history = stats_history.processed
50
49
  @failed_history = stats_history.failed
51
50
 
@@ -57,14 +56,14 @@ module Sidekiq
57
56
  end
58
57
 
59
58
  post "/busy" do
60
- if params['identity']
61
- p = Sidekiq::Process.new('identity' => params['identity'])
62
- p.quiet! if params['quiet']
63
- p.stop! if params['stop']
59
+ if params["identity"]
60
+ p = Sidekiq::Process.new("identity" => params["identity"])
61
+ p.quiet! if params["quiet"]
62
+ p.stop! if params["stop"]
64
63
  else
65
64
  processes.each do |pro|
66
- pro.quiet! if params['quiet']
67
- pro.stop! if params['stop']
65
+ pro.quiet! if params["quiet"]
66
+ pro.stop! if params["stop"]
68
67
  end
69
68
  end
70
69
 
@@ -82,37 +81,46 @@ module Sidekiq
82
81
 
83
82
  halt(404) unless @name
84
83
 
85
- @count = (params['count'] || 25).to_i
84
+ @count = (params["count"] || 25).to_i
86
85
  @queue = Sidekiq::Queue.new(@name)
87
- (@current_page, @total_size, @messages) = page("queue:#{@name}", params['page'], @count)
86
+ (@current_page, @total_size, @messages) = page("queue:#{@name}", params["page"], @count, reverse: params["direction"] == "asc")
88
87
  @messages = @messages.map { |msg| Sidekiq::Job.new(msg, @name) }
89
88
 
90
89
  erb(:queue)
91
90
  end
92
91
 
93
92
  post "/queues/:name" do
94
- Sidekiq::Queue.new(route_params[:name]).clear
93
+ queue = Sidekiq::Queue.new(route_params[:name])
94
+
95
+ if Sidekiq.pro? && params["pause"]
96
+ queue.pause!
97
+ elsif Sidekiq.pro? && params["unpause"]
98
+ queue.unpause!
99
+ else
100
+ queue.clear
101
+ end
95
102
 
96
103
  redirect "#{root_path}queues"
97
104
  end
98
105
 
99
106
  post "/queues/:name/delete" do
100
107
  name = route_params[:name]
101
- Sidekiq::Job.new(params['key_val'], name).delete
108
+ Sidekiq::Job.new(params["key_val"], name).delete
102
109
 
103
110
  redirect_with_query("#{root_path}queues/#{CGI.escape(name)}")
104
111
  end
105
112
 
106
- get '/morgue' do
107
- @count = (params['count'] || 25).to_i
108
- (@current_page, @total_size, @dead) = page("dead", params['page'], @count, reverse: true)
113
+ get "/morgue" do
114
+ @count = (params["count"] || 25).to_i
115
+ (@current_page, @total_size, @dead) = page("dead", params["page"], @count, reverse: true)
109
116
  @dead = @dead.map { |msg, score| Sidekiq::SortedEntry.new(nil, score, msg) }
110
117
 
111
118
  erb(:morgue)
112
119
  end
113
120
 
114
121
  get "/morgue/:key" do
115
- halt(404) unless key = route_params[:key]
122
+ key = route_params[:key]
123
+ halt(404) unless key
116
124
 
117
125
  @dead = Sidekiq::DeadSet.new.fetch(*parse_params(key)).first
118
126
 
@@ -123,10 +131,10 @@ module Sidekiq
123
131
  end
124
132
  end
125
133
 
126
- post '/morgue' do
127
- redirect(request.path) unless params['key']
134
+ post "/morgue" do
135
+ redirect(request.path) unless params["key"]
128
136
 
129
- params['key'].each do |key|
137
+ params["key"].each do |key|
130
138
  job = Sidekiq::DeadSet.new.fetch(*parse_params(key)).first
131
139
  retry_or_delete_or_kill job, params if job
132
140
  end
@@ -147,7 +155,8 @@ module Sidekiq
147
155
  end
148
156
 
149
157
  post "/morgue/:key" do
150
- halt(404) unless key = route_params[:key]
158
+ key = route_params[:key]
159
+ halt(404) unless key
151
160
 
152
161
  job = Sidekiq::DeadSet.new.fetch(*parse_params(key)).first
153
162
  retry_or_delete_or_kill job, params if job
@@ -155,9 +164,9 @@ module Sidekiq
155
164
  redirect_with_query("#{root_path}morgue")
156
165
  end
157
166
 
158
- get '/retries' do
159
- @count = (params['count'] || 25).to_i
160
- (@current_page, @total_size, @retries) = page("retry", params['page'], @count)
167
+ get "/retries" do
168
+ @count = (params["count"] || 25).to_i
169
+ (@current_page, @total_size, @retries) = page("retry", params["page"], @count)
161
170
  @retries = @retries.map { |msg, score| Sidekiq::SortedEntry.new(nil, score, msg) }
162
171
 
163
172
  erb(:retries)
@@ -173,10 +182,10 @@ module Sidekiq
173
182
  end
174
183
  end
175
184
 
176
- post '/retries' do
177
- redirect(request.path) unless params['key']
185
+ post "/retries" do
186
+ redirect(request.path) unless params["key"]
178
187
 
179
- params['key'].each do |key|
188
+ params["key"].each do |key|
180
189
  job = Sidekiq::RetrySet.new.fetch(*parse_params(key)).first
181
190
  retry_or_delete_or_kill job, params if job
182
191
  end
@@ -196,6 +205,12 @@ module Sidekiq
196
205
  redirect "#{root_path}retries"
197
206
  end
198
207
 
208
+ post "/retries/all/kill" do
209
+ Sidekiq::RetrySet.new.kill_all
210
+
211
+ redirect "#{root_path}retries"
212
+ end
213
+
199
214
  post "/retries/:key" do
200
215
  job = Sidekiq::RetrySet.new.fetch(*parse_params(route_params[:key])).first
201
216
 
@@ -204,9 +219,9 @@ module Sidekiq
204
219
  redirect_with_query("#{root_path}retries")
205
220
  end
206
221
 
207
- get '/scheduled' do
208
- @count = (params['count'] || 25).to_i
209
- (@current_page, @total_size, @scheduled) = page("schedule", params['page'], @count)
222
+ get "/scheduled" do
223
+ @count = (params["count"] || 25).to_i
224
+ (@current_page, @total_size, @scheduled) = page("schedule", params["page"], @count)
210
225
  @scheduled = @scheduled.map { |msg, score| Sidekiq::SortedEntry.new(nil, score, msg) }
211
226
 
212
227
  erb(:scheduled)
@@ -222,10 +237,10 @@ module Sidekiq
222
237
  end
223
238
  end
224
239
 
225
- post '/scheduled' do
226
- redirect(request.path) unless params['key']
240
+ post "/scheduled" do
241
+ redirect(request.path) unless params["key"]
227
242
 
228
- params['key'].each do |key|
243
+ params["key"].each do |key|
229
244
  job = Sidekiq::ScheduledSet.new.fetch(*parse_params(key)).first
230
245
  delete_or_add_queue job, params if job
231
246
  end
@@ -234,7 +249,8 @@ module Sidekiq
234
249
  end
235
250
 
236
251
  post "/scheduled/:key" do
237
- halt(404) unless key = route_params[:key]
252
+ key = route_params[:key]
253
+ halt(404) unless key
238
254
 
239
255
  job = Sidekiq::ScheduledSet.new.fetch(*parse_params(key)).first
240
256
  delete_or_add_queue job, params if job
@@ -242,23 +258,23 @@ module Sidekiq
242
258
  redirect_with_query("#{root_path}scheduled")
243
259
  end
244
260
 
245
- get '/dashboard/stats' do
261
+ get "/dashboard/stats" do
246
262
  redirect "#{root_path}stats"
247
263
  end
248
264
 
249
- get '/stats' do
265
+ get "/stats" do
250
266
  sidekiq_stats = Sidekiq::Stats.new
251
- redis_stats = redis_info.select { |k, v| REDIS_KEYS.include? k }
267
+ redis_stats = redis_info.select { |k, v| REDIS_KEYS.include? k }
252
268
  json(
253
269
  sidekiq: {
254
- processed: sidekiq_stats.processed,
255
- failed: sidekiq_stats.failed,
256
- busy: sidekiq_stats.workers_size,
257
- processes: sidekiq_stats.processes_size,
258
- enqueued: sidekiq_stats.enqueued,
259
- scheduled: sidekiq_stats.scheduled_size,
260
- retries: sidekiq_stats.retry_size,
261
- dead: sidekiq_stats.dead_size,
270
+ processed: sidekiq_stats.processed,
271
+ failed: sidekiq_stats.failed,
272
+ busy: sidekiq_stats.workers_size,
273
+ processes: sidekiq_stats.processes_size,
274
+ enqueued: sidekiq_stats.enqueued,
275
+ scheduled: sidekiq_stats.scheduled_size,
276
+ retries: sidekiq_stats.retry_size,
277
+ dead: sidekiq_stats.dead_size,
262
278
  default_latency: sidekiq_stats.default_queue_latency
263
279
  },
264
280
  redis: redis_stats,
@@ -266,48 +282,42 @@ module Sidekiq
266
282
  )
267
283
  end
268
284
 
269
- get '/stats/queues' do
285
+ get "/stats/queues" do
270
286
  json Sidekiq::Stats::Queues.new.lengths
271
287
  end
272
288
 
273
289
  def call(env)
274
290
  action = self.class.match(env)
275
- return [404, {"Content-Type" => "text/plain", "X-Cascade" => "pass" }, ["Not Found"]] unless action
291
+ return [404, {"Content-Type" => "text/plain", "X-Cascade" => "pass"}, ["Not Found"]] unless action
276
292
 
277
- resp = catch(:halt) do
278
- app = @klass
293
+ app = @klass
294
+ resp = catch(:halt) do # rubocop:disable Standard/SemanticBlocks
279
295
  self.class.run_befores(app, action)
280
- begin
281
- resp = action.instance_exec env, &action.block
282
- ensure
283
- self.class.run_afters(app, action)
284
- end
285
-
286
- resp
296
+ action.instance_exec env, &action.block
297
+ ensure
298
+ self.class.run_afters(app, action)
287
299
  end
288
300
 
289
301
  resp = case resp
290
302
  when Array
303
+ # redirects go here
291
304
  resp
292
305
  else
306
+ # rendered content goes here
293
307
  headers = {
294
308
  "Content-Type" => "text/html",
295
309
  "Cache-Control" => "no-cache",
296
310
  "Content-Language" => action.locale,
297
311
  "Content-Security-Policy" => CSP_HEADER
298
312
  }
299
-
313
+ # we'll let Rack calculate Content-Length for us.
300
314
  [200, headers, [resp]]
301
315
  end
302
316
 
303
- resp[1] = resp[1].dup
304
-
305
- resp[1][CONTENT_LENGTH] = resp[2].inject(0) { |l, p| l + p.bytesize }.to_s
306
-
307
317
  resp
308
318
  end
309
319
 
310
- def self.helpers(mod=nil, &block)
320
+ def self.helpers(mod = nil, &block)
311
321
  if block_given?
312
322
  WebAction.class_eval(&block)
313
323
  else
@@ -315,11 +325,11 @@ module Sidekiq
315
325
  end
316
326
  end
317
327
 
318
- def self.before(path=nil, &block)
328
+ def self.before(path = nil, &block)
319
329
  befores << [path && Regexp.new("\\A#{path.gsub("*", ".*")}\\z"), block]
320
330
  end
321
331
 
322
- def self.after(path=nil, &block)
332
+ def self.after(path = nil, &block)
323
333
  afters << [path && Regexp.new("\\A#{path.gsub("*", ".*")}\\z"), block]
324
334
  end
325
335
 
@@ -332,8 +342,8 @@ module Sidekiq
332
342
  end
333
343
 
334
344
  def self.run_hooks(hooks, app, action)
335
- hooks.select { |p,_| !p || p =~ action.env[WebRouter::PATH_INFO] }.
336
- each {|_,b| action.instance_exec(action.env, app, &b) }
345
+ hooks.select { |p, _| !p || p =~ action.env[WebRouter::PATH_INFO] }
346
+ .each { |_, b| action.instance_exec(action.env, app, &b) }
337
347
  end
338
348
 
339
349
  def self.befores