sidekiq 5.1.3 → 6.4.1
Sign up to get free protection for your applications and to get access to all the features.
Potentially problematic release.
This version of sidekiq might be problematic. Click here for more details.
- checksums.yaml +5 -5
- data/Changes.md +391 -1
- data/LICENSE +3 -3
- data/README.md +23 -34
- data/bin/sidekiq +27 -3
- data/bin/sidekiqload +68 -62
- data/bin/sidekiqmon +8 -0
- data/lib/generators/sidekiq/job_generator.rb +57 -0
- data/lib/generators/sidekiq/templates/{worker.rb.erb → job.rb.erb} +2 -2
- data/lib/generators/sidekiq/templates/{worker_spec.rb.erb → job_spec.rb.erb} +1 -1
- data/lib/generators/sidekiq/templates/{worker_test.rb.erb → job_test.rb.erb} +1 -1
- data/lib/sidekiq/api.rb +344 -243
- data/lib/sidekiq/cli.rb +209 -221
- data/lib/sidekiq/client.rb +62 -64
- data/lib/sidekiq/delay.rb +7 -6
- data/lib/sidekiq/exception_handler.rb +10 -12
- data/lib/sidekiq/extensions/action_mailer.rb +13 -22
- data/lib/sidekiq/extensions/active_record.rb +13 -10
- data/lib/sidekiq/extensions/class_methods.rb +14 -11
- data/lib/sidekiq/extensions/generic_proxy.rb +6 -4
- data/lib/sidekiq/fetch.rb +40 -32
- data/lib/sidekiq/job.rb +13 -0
- data/lib/sidekiq/job_logger.rb +35 -9
- data/lib/sidekiq/job_retry.rb +88 -68
- data/lib/sidekiq/job_util.rb +65 -0
- data/lib/sidekiq/launcher.rb +170 -73
- data/lib/sidekiq/logger.rb +170 -0
- data/lib/sidekiq/manager.rb +18 -22
- data/lib/sidekiq/middleware/chain.rb +20 -8
- data/lib/sidekiq/middleware/current_attributes.rb +57 -0
- data/lib/sidekiq/middleware/i18n.rb +5 -7
- data/lib/sidekiq/monitor.rb +133 -0
- data/lib/sidekiq/paginator.rb +20 -16
- data/lib/sidekiq/processor.rb +135 -83
- data/lib/sidekiq/rails.rb +42 -38
- data/lib/sidekiq/redis_connection.rb +49 -30
- data/lib/sidekiq/scheduled.rb +94 -31
- data/lib/sidekiq/sd_notify.rb +149 -0
- data/lib/sidekiq/systemd.rb +24 -0
- data/lib/sidekiq/testing/inline.rb +2 -1
- data/lib/sidekiq/testing.rb +40 -31
- data/lib/sidekiq/util.rb +57 -15
- data/lib/sidekiq/version.rb +2 -1
- data/lib/sidekiq/web/action.rb +15 -11
- data/lib/sidekiq/web/application.rb +109 -74
- data/lib/sidekiq/web/csrf_protection.rb +180 -0
- data/lib/sidekiq/web/helpers.rb +117 -93
- data/lib/sidekiq/web/router.rb +23 -19
- data/lib/sidekiq/web.rb +61 -105
- data/lib/sidekiq/worker.rb +257 -99
- data/lib/sidekiq.rb +81 -46
- data/sidekiq.gemspec +23 -23
- data/web/assets/images/apple-touch-icon.png +0 -0
- data/web/assets/javascripts/application.js +83 -64
- data/web/assets/javascripts/dashboard.js +66 -75
- data/web/assets/stylesheets/application-dark.css +143 -0
- data/web/assets/stylesheets/application-rtl.css +0 -4
- data/web/assets/stylesheets/application.css +77 -231
- data/web/assets/stylesheets/bootstrap.css +2 -2
- data/web/locales/ar.yml +9 -2
- data/web/locales/de.yml +14 -2
- data/web/locales/en.yml +7 -1
- data/web/locales/es.yml +21 -5
- data/web/locales/fr.yml +10 -3
- data/web/locales/ja.yml +7 -1
- data/web/locales/lt.yml +83 -0
- data/web/locales/pl.yml +4 -4
- data/web/locales/ru.yml +4 -0
- data/web/locales/vi.yml +83 -0
- data/web/views/_footer.erb +1 -1
- data/web/views/_job_info.erb +3 -2
- data/web/views/_nav.erb +3 -17
- data/web/views/_poll_link.erb +2 -5
- data/web/views/_summary.erb +7 -7
- data/web/views/busy.erb +54 -20
- data/web/views/dashboard.erb +22 -14
- data/web/views/dead.erb +3 -3
- data/web/views/layout.erb +4 -2
- data/web/views/morgue.erb +9 -6
- data/web/views/queue.erb +20 -10
- data/web/views/queues.erb +11 -3
- data/web/views/retries.erb +14 -7
- data/web/views/retry.erb +3 -3
- data/web/views/scheduled.erb +5 -2
- metadata +39 -136
- data/.github/contributing.md +0 -32
- data/.github/issue_template.md +0 -11
- data/.gitignore +0 -13
- data/.travis.yml +0 -14
- data/3.0-Upgrade.md +0 -70
- data/4.0-Upgrade.md +0 -53
- data/5.0-Upgrade.md +0 -56
- data/COMM-LICENSE +0 -95
- data/Ent-Changes.md +0 -216
- data/Gemfile +0 -8
- data/Pro-2.0-Upgrade.md +0 -138
- data/Pro-3.0-Upgrade.md +0 -44
- data/Pro-4.0-Upgrade.md +0 -35
- data/Pro-Changes.md +0 -729
- data/Rakefile +0 -8
- data/bin/sidekiqctl +0 -99
- data/code_of_conduct.md +0 -50
- data/lib/generators/sidekiq/worker_generator.rb +0 -49
- data/lib/sidekiq/core_ext.rb +0 -1
- data/lib/sidekiq/logging.rb +0 -122
- data/lib/sidekiq/middleware/server/active_record.rb +0 -23
data/lib/sidekiq/testing.rb
CHANGED
@@ -1,16 +1,16 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
|
-
require 'securerandom'
|
3
|
-
require 'sidekiq'
|
4
2
|
|
5
|
-
|
3
|
+
require "securerandom"
|
4
|
+
require "sidekiq"
|
6
5
|
|
6
|
+
module Sidekiq
|
7
7
|
class Testing
|
8
8
|
class << self
|
9
9
|
attr_accessor :__test_mode
|
10
10
|
|
11
11
|
def __set_test_mode(mode)
|
12
12
|
if block_given?
|
13
|
-
current_mode =
|
13
|
+
current_mode = __test_mode
|
14
14
|
begin
|
15
15
|
self.__test_mode = mode
|
16
16
|
yield
|
@@ -35,19 +35,19 @@ module Sidekiq
|
|
35
35
|
end
|
36
36
|
|
37
37
|
def enabled?
|
38
|
-
|
38
|
+
__test_mode != :disable
|
39
39
|
end
|
40
40
|
|
41
41
|
def disabled?
|
42
|
-
|
42
|
+
__test_mode == :disable
|
43
43
|
end
|
44
44
|
|
45
45
|
def fake?
|
46
|
-
|
46
|
+
__test_mode == :fake
|
47
47
|
end
|
48
48
|
|
49
49
|
def inline?
|
50
|
-
|
50
|
+
__test_mode == :inline
|
51
51
|
end
|
52
52
|
|
53
53
|
def server_middleware
|
@@ -57,7 +57,7 @@ module Sidekiq
|
|
57
57
|
end
|
58
58
|
|
59
59
|
def constantize(str)
|
60
|
-
names = str.split(
|
60
|
+
names = str.split("::")
|
61
61
|
names.shift if names.empty? || names.first.empty?
|
62
62
|
|
63
63
|
names.inject(Object) do |constant, name|
|
@@ -72,31 +72,31 @@ module Sidekiq
|
|
72
72
|
|
73
73
|
class EmptyQueueError < RuntimeError; end
|
74
74
|
|
75
|
-
|
76
|
-
alias_method :raw_push_real, :raw_push
|
77
|
-
|
75
|
+
module TestingClient
|
78
76
|
def raw_push(payloads)
|
79
77
|
if Sidekiq::Testing.fake?
|
80
78
|
payloads.each do |job|
|
81
79
|
job = Sidekiq.load_json(Sidekiq.dump_json(job))
|
82
|
-
job
|
83
|
-
Queues.push(job[
|
80
|
+
job["enqueued_at"] = Time.now.to_f unless job["at"]
|
81
|
+
Queues.push(job["queue"], job["class"], job)
|
84
82
|
end
|
85
83
|
true
|
86
84
|
elsif Sidekiq::Testing.inline?
|
87
85
|
payloads.each do |job|
|
88
|
-
klass = Sidekiq::Testing.constantize(job[
|
89
|
-
job[
|
86
|
+
klass = Sidekiq::Testing.constantize(job["class"])
|
87
|
+
job["id"] ||= SecureRandom.hex(12)
|
90
88
|
job_hash = Sidekiq.load_json(Sidekiq.dump_json(job))
|
91
89
|
klass.process_job(job_hash)
|
92
90
|
end
|
93
91
|
true
|
94
92
|
else
|
95
|
-
|
93
|
+
super
|
96
94
|
end
|
97
95
|
end
|
98
96
|
end
|
99
97
|
|
98
|
+
Sidekiq::Client.prepend TestingClient
|
99
|
+
|
100
100
|
module Queues
|
101
101
|
##
|
102
102
|
# The Queues class is only for testing the fake queue implementation.
|
@@ -255,27 +255,26 @@ module Sidekiq
|
|
255
255
|
# Then I should receive a welcome email to "foo@example.com"
|
256
256
|
#
|
257
257
|
module ClassMethods
|
258
|
-
|
259
258
|
# Queue for this worker
|
260
259
|
def queue
|
261
|
-
|
260
|
+
get_sidekiq_options["queue"]
|
262
261
|
end
|
263
262
|
|
264
263
|
# Jobs queued for this worker
|
265
264
|
def jobs
|
266
|
-
Queues.jobs_by_worker[
|
265
|
+
Queues.jobs_by_worker[to_s]
|
267
266
|
end
|
268
267
|
|
269
268
|
# Clear all jobs for this worker
|
270
269
|
def clear
|
271
|
-
Queues.clear_for(queue,
|
270
|
+
Queues.clear_for(queue, to_s)
|
272
271
|
end
|
273
272
|
|
274
273
|
# Drain and run all jobs for this worker
|
275
274
|
def drain
|
276
275
|
while jobs.any?
|
277
276
|
next_job = jobs.first
|
278
|
-
Queues.delete_for(next_job["jid"], next_job["queue"],
|
277
|
+
Queues.delete_for(next_job["jid"], next_job["queue"], to_s)
|
279
278
|
process_job(next_job)
|
280
279
|
end
|
281
280
|
end
|
@@ -284,16 +283,16 @@ module Sidekiq
|
|
284
283
|
def perform_one
|
285
284
|
raise(EmptyQueueError, "perform_one called with empty job queue") if jobs.empty?
|
286
285
|
next_job = jobs.first
|
287
|
-
Queues.delete_for(next_job["jid"], queue,
|
286
|
+
Queues.delete_for(next_job["jid"], queue, to_s)
|
288
287
|
process_job(next_job)
|
289
288
|
end
|
290
289
|
|
291
290
|
def process_job(job)
|
292
291
|
worker = new
|
293
|
-
worker.jid = job[
|
294
|
-
worker.bid = job[
|
295
|
-
Sidekiq::Testing.server_middleware.invoke(worker, job, job[
|
296
|
-
execute_job(worker, job[
|
292
|
+
worker.jid = job["jid"]
|
293
|
+
worker.bid = job["bid"] if worker.respond_to?(:bid=)
|
294
|
+
Sidekiq::Testing.server_middleware.invoke(worker, job, job["queue"]) do
|
295
|
+
execute_job(worker, job["args"])
|
297
296
|
end
|
298
297
|
end
|
299
298
|
|
@@ -324,10 +323,20 @@ module Sidekiq
|
|
324
323
|
end
|
325
324
|
end
|
326
325
|
end
|
326
|
+
|
327
|
+
module TestingExtensions
|
328
|
+
def jobs_for(klass)
|
329
|
+
jobs.select do |job|
|
330
|
+
marshalled = job["args"][0]
|
331
|
+
marshalled.index(klass.to_s) && YAML.load(marshalled)[0] == klass
|
332
|
+
end
|
333
|
+
end
|
334
|
+
end
|
335
|
+
|
336
|
+
Sidekiq::Extensions::DelayedMailer.extend(TestingExtensions) if defined?(Sidekiq::Extensions::DelayedMailer)
|
337
|
+
Sidekiq::Extensions::DelayedModel.extend(TestingExtensions) if defined?(Sidekiq::Extensions::DelayedModel)
|
327
338
|
end
|
328
339
|
|
329
|
-
if defined?(::Rails) && Rails.respond_to?(:env) && !Rails.env.test?
|
330
|
-
|
331
|
-
puts("⛔️ WARNING: Sidekiq testing API enabled, but this is not the test environment. Your jobs will not go to Redis.")
|
332
|
-
puts("**************************************************")
|
340
|
+
if defined?(::Rails) && Rails.respond_to?(:env) && !Rails.env.test? && !$TESTING
|
341
|
+
warn("⛔️ WARNING: Sidekiq testing API enabled, but this is not the test environment. Your jobs will not go to Redis.", uplevel: 1)
|
333
342
|
end
|
data/lib/sidekiq/util.rb
CHANGED
@@ -1,27 +1,67 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
|
-
|
3
|
-
require
|
4
|
-
require
|
2
|
+
|
3
|
+
require "forwardable"
|
4
|
+
require "socket"
|
5
|
+
require "securerandom"
|
6
|
+
require "sidekiq/exception_handler"
|
5
7
|
|
6
8
|
module Sidekiq
|
7
9
|
##
|
8
10
|
# This module is part of Sidekiq core and not intended for extensions.
|
9
11
|
#
|
12
|
+
|
13
|
+
class RingBuffer
|
14
|
+
include Enumerable
|
15
|
+
extend Forwardable
|
16
|
+
def_delegators :@buf, :[], :each, :size
|
17
|
+
|
18
|
+
def initialize(size, default = 0)
|
19
|
+
@size = size
|
20
|
+
@buf = Array.new(size, default)
|
21
|
+
@index = 0
|
22
|
+
end
|
23
|
+
|
24
|
+
def <<(element)
|
25
|
+
@buf[@index % @size] = element
|
26
|
+
@index += 1
|
27
|
+
element
|
28
|
+
end
|
29
|
+
|
30
|
+
def buffer
|
31
|
+
@buf
|
32
|
+
end
|
33
|
+
|
34
|
+
def reset(default = 0)
|
35
|
+
@buf.fill(default)
|
36
|
+
end
|
37
|
+
end
|
38
|
+
|
10
39
|
module Util
|
11
40
|
include ExceptionHandler
|
12
41
|
|
13
|
-
|
42
|
+
# hack for quicker development / testing environment #2774
|
43
|
+
PAUSE_TIME = $stdout.tty? ? 0.1 : 0.5
|
44
|
+
|
45
|
+
# Wait for the orblock to be true or the deadline passed.
|
46
|
+
def wait_for(deadline, &condblock)
|
47
|
+
remaining = deadline - ::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
|
48
|
+
while remaining > PAUSE_TIME
|
49
|
+
return if condblock.call
|
50
|
+
sleep PAUSE_TIME
|
51
|
+
remaining = deadline - ::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
|
52
|
+
end
|
53
|
+
end
|
14
54
|
|
15
55
|
def watchdog(last_words)
|
16
56
|
yield
|
17
57
|
rescue Exception => ex
|
18
|
-
handle_exception(ex, {
|
58
|
+
handle_exception(ex, {context: last_words})
|
19
59
|
raise ex
|
20
60
|
end
|
21
61
|
|
22
62
|
def safe_thread(name, &block)
|
23
63
|
Thread.new do
|
24
|
-
Thread.current
|
64
|
+
Thread.current.name = name
|
25
65
|
watchdog(name, &block)
|
26
66
|
end
|
27
67
|
end
|
@@ -34,8 +74,12 @@ module Sidekiq
|
|
34
74
|
Sidekiq.redis(&block)
|
35
75
|
end
|
36
76
|
|
77
|
+
def tid
|
78
|
+
Thread.current["sidekiq_tid"] ||= (Thread.current.object_id ^ ::Process.pid).to_s(36)
|
79
|
+
end
|
80
|
+
|
37
81
|
def hostname
|
38
|
-
ENV[
|
82
|
+
ENV["DYNO"] || Socket.gethostname
|
39
83
|
end
|
40
84
|
|
41
85
|
def process_nonce
|
@@ -43,22 +87,20 @@ module Sidekiq
|
|
43
87
|
end
|
44
88
|
|
45
89
|
def identity
|
46
|
-
@@identity ||= "#{hostname}:#{
|
90
|
+
@@identity ||= "#{hostname}:#{::Process.pid}:#{process_nonce}"
|
47
91
|
end
|
48
92
|
|
49
|
-
def fire_event(event, options={})
|
93
|
+
def fire_event(event, options = {})
|
50
94
|
reverse = options[:reverse]
|
51
95
|
reraise = options[:reraise]
|
52
96
|
|
53
97
|
arr = Sidekiq.options[:lifecycle_events][event]
|
54
98
|
arr.reverse! if reverse
|
55
99
|
arr.each do |block|
|
56
|
-
|
57
|
-
|
58
|
-
|
59
|
-
|
60
|
-
raise ex if reraise
|
61
|
-
end
|
100
|
+
block.call
|
101
|
+
rescue => ex
|
102
|
+
handle_exception(ex, {context: "Exception during Sidekiq lifecycle event.", event: event})
|
103
|
+
raise ex if reraise
|
62
104
|
end
|
63
105
|
arr.clear
|
64
106
|
end
|
data/lib/sidekiq/version.rb
CHANGED
data/lib/sidekiq/web/action.rb
CHANGED
@@ -2,7 +2,7 @@
|
|
2
2
|
|
3
3
|
module Sidekiq
|
4
4
|
class WebAction
|
5
|
-
RACK_SESSION =
|
5
|
+
RACK_SESSION = "rack.session"
|
6
6
|
|
7
7
|
attr_accessor :env, :block, :type
|
8
8
|
|
@@ -15,18 +15,18 @@ module Sidekiq
|
|
15
15
|
end
|
16
16
|
|
17
17
|
def halt(res)
|
18
|
-
throw :halt, res
|
18
|
+
throw :halt, [res, {"Content-Type" => "text/plain"}, [res.to_s]]
|
19
19
|
end
|
20
20
|
|
21
21
|
def redirect(location)
|
22
|
-
throw :halt, [302, {
|
22
|
+
throw :halt, [302, {"Location" => "#{request.base_url}#{location}"}, []]
|
23
23
|
end
|
24
24
|
|
25
25
|
def params
|
26
|
-
indifferent_hash = Hash.new {|hash,key| hash[key.to_s] if Symbol === key }
|
26
|
+
indifferent_hash = Hash.new { |hash, key| hash[key.to_s] if Symbol === key }
|
27
27
|
|
28
28
|
indifferent_hash.merge! request.params
|
29
|
-
route_params.each {|k,v| indifferent_hash[k.to_s] = v }
|
29
|
+
route_params.each { |k, v| indifferent_hash[k.to_s] = v }
|
30
30
|
|
31
31
|
indifferent_hash
|
32
32
|
end
|
@@ -40,10 +40,14 @@ module Sidekiq
|
|
40
40
|
end
|
41
41
|
|
42
42
|
def erb(content, options = {})
|
43
|
-
if content.
|
43
|
+
if content.is_a? Symbol
|
44
44
|
unless respond_to?(:"_erb_#{content}")
|
45
45
|
src = ERB.new(File.read("#{Web.settings.views}/#{content}.erb")).src
|
46
|
-
WebAction.class_eval
|
46
|
+
WebAction.class_eval <<-RUBY, __FILE__, __LINE__ + 1
|
47
|
+
def _erb_#{content}
|
48
|
+
#{src}
|
49
|
+
end
|
50
|
+
RUBY
|
47
51
|
end
|
48
52
|
end
|
49
53
|
|
@@ -64,22 +68,22 @@ module Sidekiq
|
|
64
68
|
end
|
65
69
|
|
66
70
|
def json(payload)
|
67
|
-
[200, {
|
71
|
+
[200, {"Content-Type" => "application/json", "Cache-Control" => "private, no-store"}, [Sidekiq.dump_json(payload)]]
|
68
72
|
end
|
69
73
|
|
70
74
|
def initialize(env, block)
|
71
75
|
@_erb = false
|
72
76
|
@env = env
|
73
77
|
@block = block
|
74
|
-
|
78
|
+
@files ||= {}
|
75
79
|
end
|
76
80
|
|
77
81
|
private
|
78
82
|
|
79
83
|
def _erb(file, locals)
|
80
|
-
locals
|
84
|
+
locals&.each { |k, v| define_singleton_method(k) { v } unless singleton_methods.include? k }
|
81
85
|
|
82
|
-
if file.
|
86
|
+
if file.is_a?(String)
|
83
87
|
ERB.new(file).result(binding)
|
84
88
|
else
|
85
89
|
send(:"_erb_#{file}")
|