sidekiq 5.2.7 → 6.0.3
Sign up to get free protection for your applications and to get access to all the features.
Potentially problematic release.
This version of sidekiq might be problematic. Click here for more details.
- checksums.yaml +4 -4
- data/.circleci/config.yml +21 -0
- data/.gitignore +0 -2
- data/.standard.yml +20 -0
- data/6.0-Upgrade.md +72 -0
- data/Changes.md +121 -0
- data/Ent-2.0-Upgrade.md +37 -0
- data/Ent-Changes.md +18 -0
- data/Gemfile +12 -11
- data/Gemfile.lock +196 -0
- data/Pro-5.0-Upgrade.md +25 -0
- data/Pro-Changes.md +18 -1
- data/README.md +18 -30
- data/Rakefile +5 -4
- data/bin/sidekiqload +32 -24
- data/bin/sidekiqmon +8 -0
- data/lib/generators/sidekiq/templates/worker_test.rb.erb +1 -1
- data/lib/generators/sidekiq/worker_generator.rb +20 -12
- data/lib/sidekiq.rb +61 -42
- data/lib/sidekiq/api.rb +230 -214
- data/lib/sidekiq/cli.rb +111 -174
- data/lib/sidekiq/client.rb +55 -46
- data/lib/sidekiq/delay.rb +5 -6
- data/lib/sidekiq/exception_handler.rb +10 -12
- data/lib/sidekiq/extensions/action_mailer.rb +10 -20
- data/lib/sidekiq/extensions/active_record.rb +9 -7
- data/lib/sidekiq/extensions/class_methods.rb +9 -7
- data/lib/sidekiq/extensions/generic_proxy.rb +4 -4
- data/lib/sidekiq/fetch.rb +11 -12
- data/lib/sidekiq/job_logger.rb +45 -7
- data/lib/sidekiq/job_retry.rb +60 -60
- data/lib/sidekiq/launcher.rb +57 -51
- data/lib/sidekiq/logger.rb +165 -0
- data/lib/sidekiq/manager.rb +7 -9
- data/lib/sidekiq/middleware/chain.rb +14 -4
- data/lib/sidekiq/middleware/i18n.rb +5 -7
- data/lib/sidekiq/monitor.rb +133 -0
- data/lib/sidekiq/paginator.rb +18 -14
- data/lib/sidekiq/processor.rb +67 -66
- data/lib/sidekiq/rails.rb +23 -29
- data/lib/sidekiq/redis_connection.rb +31 -37
- data/lib/sidekiq/scheduled.rb +28 -29
- data/lib/sidekiq/testing.rb +34 -23
- data/lib/sidekiq/testing/inline.rb +2 -1
- data/lib/sidekiq/util.rb +17 -16
- data/lib/sidekiq/version.rb +2 -1
- data/lib/sidekiq/web.rb +41 -49
- data/lib/sidekiq/web/action.rb +14 -10
- data/lib/sidekiq/web/application.rb +64 -66
- data/lib/sidekiq/web/helpers.rb +83 -72
- data/lib/sidekiq/web/router.rb +17 -14
- data/lib/sidekiq/worker.rb +124 -97
- data/sidekiq.gemspec +16 -16
- data/web/assets/javascripts/dashboard.js +4 -23
- data/web/assets/stylesheets/application-dark.css +125 -0
- data/web/assets/stylesheets/application.css +9 -0
- data/web/locales/de.yml +14 -2
- data/web/locales/ja.yml +2 -1
- data/web/views/_job_info.erb +2 -1
- data/web/views/busy.erb +4 -1
- data/web/views/dead.erb +2 -2
- data/web/views/layout.erb +1 -0
- data/web/views/morgue.erb +4 -1
- data/web/views/queue.erb +10 -1
- data/web/views/retries.erb +4 -1
- data/web/views/retry.erb +2 -2
- data/web/views/scheduled.erb +4 -1
- metadata +20 -29
- data/.travis.yml +0 -11
- data/bin/sidekiqctl +0 -20
- data/lib/sidekiq/core_ext.rb +0 -1
- data/lib/sidekiq/ctl.rb +0 -221
- data/lib/sidekiq/logging.rb +0 -122
- data/lib/sidekiq/middleware/server/active_record.rb +0 -23
data/lib/sidekiq/testing.rb
CHANGED
@@ -1,16 +1,16 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
|
-
require 'securerandom'
|
3
|
-
require 'sidekiq'
|
4
2
|
|
5
|
-
|
3
|
+
require "securerandom"
|
4
|
+
require "sidekiq"
|
6
5
|
|
6
|
+
module Sidekiq
|
7
7
|
class Testing
|
8
8
|
class << self
|
9
9
|
attr_accessor :__test_mode
|
10
10
|
|
11
11
|
def __set_test_mode(mode)
|
12
12
|
if block_given?
|
13
|
-
current_mode =
|
13
|
+
current_mode = __test_mode
|
14
14
|
begin
|
15
15
|
self.__test_mode = mode
|
16
16
|
yield
|
@@ -35,19 +35,19 @@ module Sidekiq
|
|
35
35
|
end
|
36
36
|
|
37
37
|
def enabled?
|
38
|
-
|
38
|
+
__test_mode != :disable
|
39
39
|
end
|
40
40
|
|
41
41
|
def disabled?
|
42
|
-
|
42
|
+
__test_mode == :disable
|
43
43
|
end
|
44
44
|
|
45
45
|
def fake?
|
46
|
-
|
46
|
+
__test_mode == :fake
|
47
47
|
end
|
48
48
|
|
49
49
|
def inline?
|
50
|
-
|
50
|
+
__test_mode == :inline
|
51
51
|
end
|
52
52
|
|
53
53
|
def server_middleware
|
@@ -57,7 +57,7 @@ module Sidekiq
|
|
57
57
|
end
|
58
58
|
|
59
59
|
def constantize(str)
|
60
|
-
names = str.split(
|
60
|
+
names = str.split("::")
|
61
61
|
names.shift if names.empty? || names.first.empty?
|
62
62
|
|
63
63
|
names.inject(Object) do |constant, name|
|
@@ -77,14 +77,14 @@ module Sidekiq
|
|
77
77
|
if Sidekiq::Testing.fake?
|
78
78
|
payloads.each do |job|
|
79
79
|
job = Sidekiq.load_json(Sidekiq.dump_json(job))
|
80
|
-
job
|
81
|
-
Queues.push(job[
|
80
|
+
job["enqueued_at"] = Time.now.to_f unless job["at"]
|
81
|
+
Queues.push(job["queue"], job["class"], job)
|
82
82
|
end
|
83
83
|
true
|
84
84
|
elsif Sidekiq::Testing.inline?
|
85
85
|
payloads.each do |job|
|
86
|
-
klass = Sidekiq::Testing.constantize(job[
|
87
|
-
job[
|
86
|
+
klass = Sidekiq::Testing.constantize(job["class"])
|
87
|
+
job["id"] ||= SecureRandom.hex(12)
|
88
88
|
job_hash = Sidekiq.load_json(Sidekiq.dump_json(job))
|
89
89
|
klass.process_job(job_hash)
|
90
90
|
end
|
@@ -255,27 +255,26 @@ module Sidekiq
|
|
255
255
|
# Then I should receive a welcome email to "foo@example.com"
|
256
256
|
#
|
257
257
|
module ClassMethods
|
258
|
-
|
259
258
|
# Queue for this worker
|
260
259
|
def queue
|
261
|
-
|
260
|
+
get_sidekiq_options["queue"]
|
262
261
|
end
|
263
262
|
|
264
263
|
# Jobs queued for this worker
|
265
264
|
def jobs
|
266
|
-
Queues.jobs_by_worker[
|
265
|
+
Queues.jobs_by_worker[to_s]
|
267
266
|
end
|
268
267
|
|
269
268
|
# Clear all jobs for this worker
|
270
269
|
def clear
|
271
|
-
Queues.clear_for(queue,
|
270
|
+
Queues.clear_for(queue, to_s)
|
272
271
|
end
|
273
272
|
|
274
273
|
# Drain and run all jobs for this worker
|
275
274
|
def drain
|
276
275
|
while jobs.any?
|
277
276
|
next_job = jobs.first
|
278
|
-
Queues.delete_for(next_job["jid"], next_job["queue"],
|
277
|
+
Queues.delete_for(next_job["jid"], next_job["queue"], to_s)
|
279
278
|
process_job(next_job)
|
280
279
|
end
|
281
280
|
end
|
@@ -284,16 +283,16 @@ module Sidekiq
|
|
284
283
|
def perform_one
|
285
284
|
raise(EmptyQueueError, "perform_one called with empty job queue") if jobs.empty?
|
286
285
|
next_job = jobs.first
|
287
|
-
Queues.delete_for(next_job["jid"], queue,
|
286
|
+
Queues.delete_for(next_job["jid"], queue, to_s)
|
288
287
|
process_job(next_job)
|
289
288
|
end
|
290
289
|
|
291
290
|
def process_job(job)
|
292
291
|
worker = new
|
293
|
-
worker.jid = job[
|
294
|
-
worker.bid = job[
|
295
|
-
Sidekiq::Testing.server_middleware.invoke(worker, job, job[
|
296
|
-
execute_job(worker, job[
|
292
|
+
worker.jid = job["jid"]
|
293
|
+
worker.bid = job["bid"] if worker.respond_to?(:bid=)
|
294
|
+
Sidekiq::Testing.server_middleware.invoke(worker, job, job["queue"]) do
|
295
|
+
execute_job(worker, job["args"])
|
297
296
|
end
|
298
297
|
end
|
299
298
|
|
@@ -324,6 +323,18 @@ module Sidekiq
|
|
324
323
|
end
|
325
324
|
end
|
326
325
|
end
|
326
|
+
|
327
|
+
module TestingExtensions
|
328
|
+
def jobs_for(klass)
|
329
|
+
jobs.select do |job|
|
330
|
+
marshalled = job["args"][0]
|
331
|
+
marshalled.index(klass.to_s) && YAML.load(marshalled)[0] == klass
|
332
|
+
end
|
333
|
+
end
|
334
|
+
end
|
335
|
+
|
336
|
+
Sidekiq::Extensions::DelayedMailer.extend(TestingExtensions) if defined?(Sidekiq::Extensions::DelayedMailer)
|
337
|
+
Sidekiq::Extensions::DelayedModel.extend(TestingExtensions) if defined?(Sidekiq::Extensions::DelayedModel)
|
327
338
|
end
|
328
339
|
|
329
340
|
if defined?(::Rails) && Rails.respond_to?(:env) && !Rails.env.test?
|
data/lib/sidekiq/util.rb
CHANGED
@@ -1,7 +1,8 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
|
-
|
3
|
-
require
|
4
|
-
require
|
2
|
+
|
3
|
+
require "socket"
|
4
|
+
require "securerandom"
|
5
|
+
require "sidekiq/exception_handler"
|
5
6
|
|
6
7
|
module Sidekiq
|
7
8
|
##
|
@@ -10,18 +11,16 @@ module Sidekiq
|
|
10
11
|
module Util
|
11
12
|
include ExceptionHandler
|
12
13
|
|
13
|
-
EXPIRY = 60 * 60 * 24
|
14
|
-
|
15
14
|
def watchdog(last_words)
|
16
15
|
yield
|
17
16
|
rescue Exception => ex
|
18
|
-
handle_exception(ex, {
|
17
|
+
handle_exception(ex, {context: last_words})
|
19
18
|
raise ex
|
20
19
|
end
|
21
20
|
|
22
21
|
def safe_thread(name, &block)
|
23
22
|
Thread.new do
|
24
|
-
Thread.current
|
23
|
+
Thread.current.name = name
|
25
24
|
watchdog(name, &block)
|
26
25
|
end
|
27
26
|
end
|
@@ -34,8 +33,12 @@ module Sidekiq
|
|
34
33
|
Sidekiq.redis(&block)
|
35
34
|
end
|
36
35
|
|
36
|
+
def tid
|
37
|
+
Thread.current["sidekiq_tid"] ||= (Thread.current.object_id ^ ::Process.pid).to_s(36)
|
38
|
+
end
|
39
|
+
|
37
40
|
def hostname
|
38
|
-
ENV[
|
41
|
+
ENV["DYNO"] || Socket.gethostname
|
39
42
|
end
|
40
43
|
|
41
44
|
def process_nonce
|
@@ -43,22 +46,20 @@ module Sidekiq
|
|
43
46
|
end
|
44
47
|
|
45
48
|
def identity
|
46
|
-
@@identity ||= "#{hostname}:#{
|
49
|
+
@@identity ||= "#{hostname}:#{::Process.pid}:#{process_nonce}"
|
47
50
|
end
|
48
51
|
|
49
|
-
def fire_event(event, options={})
|
52
|
+
def fire_event(event, options = {})
|
50
53
|
reverse = options[:reverse]
|
51
54
|
reraise = options[:reraise]
|
52
55
|
|
53
56
|
arr = Sidekiq.options[:lifecycle_events][event]
|
54
57
|
arr.reverse! if reverse
|
55
58
|
arr.each do |block|
|
56
|
-
|
57
|
-
|
58
|
-
|
59
|
-
|
60
|
-
raise ex if reraise
|
61
|
-
end
|
59
|
+
block.call
|
60
|
+
rescue => ex
|
61
|
+
handle_exception(ex, {context: "Exception during Sidekiq lifecycle event.", event: event})
|
62
|
+
raise ex if reraise
|
62
63
|
end
|
63
64
|
arr.clear
|
64
65
|
end
|
data/lib/sidekiq/version.rb
CHANGED
data/lib/sidekiq/web.rb
CHANGED
@@ -1,20 +1,21 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
|
-
require 'erb'
|
3
2
|
|
4
|
-
require
|
5
|
-
require 'sidekiq/api'
|
6
|
-
require 'sidekiq/paginator'
|
7
|
-
require 'sidekiq/web/helpers'
|
3
|
+
require "erb"
|
8
4
|
|
9
|
-
require
|
10
|
-
require
|
11
|
-
require
|
5
|
+
require "sidekiq"
|
6
|
+
require "sidekiq/api"
|
7
|
+
require "sidekiq/paginator"
|
8
|
+
require "sidekiq/web/helpers"
|
12
9
|
|
13
|
-
require
|
10
|
+
require "sidekiq/web/router"
|
11
|
+
require "sidekiq/web/action"
|
12
|
+
require "sidekiq/web/application"
|
14
13
|
|
15
|
-
require
|
16
|
-
|
17
|
-
require
|
14
|
+
require "rack/protection"
|
15
|
+
|
16
|
+
require "rack/builder"
|
17
|
+
require "rack/file"
|
18
|
+
require "rack/session/cookie"
|
18
19
|
|
19
20
|
module Sidekiq
|
20
21
|
class Web
|
@@ -25,12 +26,12 @@ module Sidekiq
|
|
25
26
|
ASSETS = "#{ROOT}/assets"
|
26
27
|
|
27
28
|
DEFAULT_TABS = {
|
28
|
-
"Dashboard" =>
|
29
|
-
"Busy"
|
30
|
-
"Queues"
|
31
|
-
"Retries"
|
32
|
-
"Scheduled" =>
|
33
|
-
"Dead"
|
29
|
+
"Dashboard" => "",
|
30
|
+
"Busy" => "busy",
|
31
|
+
"Queues" => "queues",
|
32
|
+
"Retries" => "retries",
|
33
|
+
"Scheduled" => "scheduled",
|
34
|
+
"Dead" => "morgue",
|
34
35
|
}
|
35
36
|
|
36
37
|
class << self
|
@@ -64,11 +65,11 @@ module Sidekiq
|
|
64
65
|
end
|
65
66
|
|
66
67
|
def enable(*opts)
|
67
|
-
opts.each {|key| set(key, true) }
|
68
|
+
opts.each { |key| set(key, true) }
|
68
69
|
end
|
69
70
|
|
70
71
|
def disable(*opts)
|
71
|
-
opts.each {|key| set(key, false) }
|
72
|
+
opts.each { |key| set(key, false) }
|
72
73
|
end
|
73
74
|
|
74
75
|
# Helper for the Sinatra syntax: Sidekiq::Web.set(:session_secret, Rails.application.secrets...)
|
@@ -81,10 +82,10 @@ module Sidekiq
|
|
81
82
|
end
|
82
83
|
|
83
84
|
def self.inherited(child)
|
84
|
-
child.app_url =
|
85
|
-
child.session_secret =
|
86
|
-
child.redis_pool =
|
87
|
-
child.sessions =
|
85
|
+
child.app_url = app_url
|
86
|
+
child.session_secret = session_secret
|
87
|
+
child.redis_pool = redis_pool
|
88
|
+
child.sessions = sessions
|
88
89
|
end
|
89
90
|
|
90
91
|
def settings
|
@@ -113,11 +114,11 @@ module Sidekiq
|
|
113
114
|
end
|
114
115
|
|
115
116
|
def enable(*opts)
|
116
|
-
opts.each {|key| set(key, true) }
|
117
|
+
opts.each { |key| set(key, true) }
|
117
118
|
end
|
118
119
|
|
119
120
|
def disable(*opts)
|
120
|
-
opts.each {|key| set(key, false) }
|
121
|
+
opts.each { |key| set(key, false) }
|
121
122
|
end
|
122
123
|
|
123
124
|
def set(attribute, value)
|
@@ -145,28 +146,28 @@ module Sidekiq
|
|
145
146
|
private
|
146
147
|
|
147
148
|
def using?(middleware)
|
148
|
-
middlewares.any? do |(m,_)|
|
149
|
-
m.
|
149
|
+
middlewares.any? do |(m, _)|
|
150
|
+
m.is_a?(Array) && (m[0] == middleware || m[0].is_a?(middleware))
|
150
151
|
end
|
151
152
|
end
|
152
153
|
|
153
154
|
def build_sessions
|
154
155
|
middlewares = self.middlewares
|
155
156
|
|
156
|
-
unless using?(::Rack::Protection) || ENV[
|
157
|
-
middlewares.unshift [[::Rack::Protection, {
|
157
|
+
unless using?(::Rack::Protection) || ENV["RACK_ENV"] == "test"
|
158
|
+
middlewares.unshift [[::Rack::Protection, {use: :authenticity_token}], nil]
|
158
159
|
end
|
159
160
|
|
160
161
|
s = sessions
|
161
162
|
return unless s
|
162
163
|
|
163
164
|
unless using? ::Rack::Session::Cookie
|
164
|
-
unless secret = Web.session_secret
|
165
|
-
require
|
165
|
+
unless (secret = Web.session_secret)
|
166
|
+
require "securerandom"
|
166
167
|
secret = SecureRandom.hex(64)
|
167
168
|
end
|
168
169
|
|
169
|
-
options = {
|
170
|
+
options = {secret: secret}
|
170
171
|
options = options.merge(s.to_hash) if s.respond_to? :to_hash
|
171
172
|
|
172
173
|
middlewares.unshift [[::Rack::Session::Cookie, options], nil]
|
@@ -180,13 +181,13 @@ module Sidekiq
|
|
180
181
|
klass = self.class
|
181
182
|
|
182
183
|
::Rack::Builder.new do
|
183
|
-
%w
|
184
|
+
%w[stylesheets javascripts images].each do |asset_dir|
|
184
185
|
map "/#{asset_dir}" do
|
185
|
-
run ::Rack::File.new("#{ASSETS}/#{asset_dir}", {
|
186
|
+
run ::Rack::File.new("#{ASSETS}/#{asset_dir}", {"Cache-Control" => "public, max-age=86400"})
|
186
187
|
end
|
187
188
|
end
|
188
189
|
|
189
|
-
middlewares.each {|middleware, block| use(*middleware, &block) }
|
190
|
+
middlewares.each { |middleware, block| use(*middleware, &block) }
|
190
191
|
|
191
192
|
run WebApplication.new(klass)
|
192
193
|
end
|
@@ -196,18 +197,9 @@ module Sidekiq
|
|
196
197
|
Sidekiq::WebApplication.helpers WebHelpers
|
197
198
|
Sidekiq::WebApplication.helpers Sidekiq::Paginator
|
198
199
|
|
199
|
-
Sidekiq::WebAction.class_eval
|
200
|
-
|
201
|
-
|
202
|
-
if defined?(::ActionDispatch::Request::Session) &&
|
203
|
-
!::ActionDispatch::Request::Session.method_defined?(:each)
|
204
|
-
# mperham/sidekiq#2460
|
205
|
-
# Rack apps can't reuse the Rails session store without
|
206
|
-
# this monkeypatch, fixed in Rails 5.
|
207
|
-
class ActionDispatch::Request::Session
|
208
|
-
def each(&block)
|
209
|
-
hash = self.to_hash
|
210
|
-
hash.each(&block)
|
200
|
+
Sidekiq::WebAction.class_eval <<-RUBY, __FILE__, __LINE__ + 1
|
201
|
+
def _render
|
202
|
+
#{ERB.new(File.read(Web::LAYOUT)).src}
|
211
203
|
end
|
212
|
-
|
204
|
+
RUBY
|
213
205
|
end
|
data/lib/sidekiq/web/action.rb
CHANGED
@@ -2,7 +2,7 @@
|
|
2
2
|
|
3
3
|
module Sidekiq
|
4
4
|
class WebAction
|
5
|
-
RACK_SESSION =
|
5
|
+
RACK_SESSION = "rack.session"
|
6
6
|
|
7
7
|
attr_accessor :env, :block, :type
|
8
8
|
|
@@ -19,14 +19,14 @@ module Sidekiq
|
|
19
19
|
end
|
20
20
|
|
21
21
|
def redirect(location)
|
22
|
-
throw :halt, [302, {
|
22
|
+
throw :halt, [302, {"Location" => "#{request.base_url}#{location}"}, []]
|
23
23
|
end
|
24
24
|
|
25
25
|
def params
|
26
|
-
indifferent_hash = Hash.new {|hash,key| hash[key.to_s] if Symbol === key }
|
26
|
+
indifferent_hash = Hash.new { |hash, key| hash[key.to_s] if Symbol === key }
|
27
27
|
|
28
28
|
indifferent_hash.merge! request.params
|
29
|
-
route_params.each {|k,v| indifferent_hash[k.to_s] = v }
|
29
|
+
route_params.each { |k, v| indifferent_hash[k.to_s] = v }
|
30
30
|
|
31
31
|
indifferent_hash
|
32
32
|
end
|
@@ -40,10 +40,14 @@ module Sidekiq
|
|
40
40
|
end
|
41
41
|
|
42
42
|
def erb(content, options = {})
|
43
|
-
if content.
|
43
|
+
if content.is_a? Symbol
|
44
44
|
unless respond_to?(:"_erb_#{content}")
|
45
45
|
src = ERB.new(File.read("#{Web.settings.views}/#{content}.erb")).src
|
46
|
-
WebAction.class_eval
|
46
|
+
WebAction.class_eval <<-RUBY, __FILE__, __LINE__ + 1
|
47
|
+
def _erb_#{content}
|
48
|
+
#{src}
|
49
|
+
end
|
50
|
+
RUBY
|
47
51
|
end
|
48
52
|
end
|
49
53
|
|
@@ -64,22 +68,22 @@ module Sidekiq
|
|
64
68
|
end
|
65
69
|
|
66
70
|
def json(payload)
|
67
|
-
[200, {
|
71
|
+
[200, {"Content-Type" => "application/json", "Cache-Control" => "no-cache"}, [Sidekiq.dump_json(payload)]]
|
68
72
|
end
|
69
73
|
|
70
74
|
def initialize(env, block)
|
71
75
|
@_erb = false
|
72
76
|
@env = env
|
73
77
|
@block = block
|
74
|
-
|
78
|
+
@files ||= {}
|
75
79
|
end
|
76
80
|
|
77
81
|
private
|
78
82
|
|
79
83
|
def _erb(file, locals)
|
80
|
-
locals
|
84
|
+
locals&.each { |k, v| define_singleton_method(k) { v } unless singleton_methods.include? k }
|
81
85
|
|
82
|
-
if file.
|
86
|
+
if file.is_a?(String)
|
83
87
|
ERB.new(file).result(binding)
|
84
88
|
else
|
85
89
|
send(:"_erb_#{file}")
|