sidekiq 6.0.0 → 6.0.1
Sign up to get free protection for your applications and to get access to all the features.
Potentially problematic release.
This version of sidekiq might be problematic. Click here for more details.
- checksums.yaml +4 -4
- data/.circleci/config.yml +21 -0
- data/6.0-Upgrade.md +3 -1
- data/Changes.md +76 -1
- data/Ent-Changes.md +6 -0
- data/Gemfile.lock +3 -3
- data/Pro-Changes.md +9 -1
- data/README.md +1 -0
- data/bin/sidekiqload +8 -4
- data/bin/sidekiqmon +4 -5
- data/lib/generators/sidekiq/worker_generator.rb +10 -0
- data/lib/sidekiq.rb +8 -0
- data/lib/sidekiq/api.rb +74 -40
- data/lib/sidekiq/cli.rb +16 -16
- data/lib/sidekiq/client.rb +8 -2
- data/lib/sidekiq/fetch.rb +7 -7
- data/lib/sidekiq/job_logger.rb +11 -3
- data/lib/sidekiq/job_retry.rb +17 -6
- data/lib/sidekiq/launcher.rb +1 -3
- data/lib/sidekiq/logger.rb +107 -11
- data/lib/sidekiq/middleware/chain.rb +11 -2
- data/lib/sidekiq/monitor.rb +2 -2
- data/lib/sidekiq/paginator.rb +7 -2
- data/lib/sidekiq/processor.rb +31 -11
- data/lib/sidekiq/scheduled.rb +13 -12
- data/lib/sidekiq/testing.rb +12 -0
- data/lib/sidekiq/version.rb +1 -1
- data/lib/sidekiq/web/application.rb +7 -11
- data/lib/sidekiq/web/helpers.rb +22 -4
- data/lib/sidekiq/worker.rb +4 -4
- data/sidekiq.gemspec +1 -1
- data/web/assets/javascripts/dashboard.js +2 -2
- data/web/assets/stylesheets/application-dark.css +125 -0
- data/web/assets/stylesheets/application.css +9 -0
- data/web/views/_job_info.erb +2 -1
- data/web/views/busy.erb +4 -1
- data/web/views/dead.erb +2 -2
- data/web/views/layout.erb +1 -0
- data/web/views/morgue.erb +4 -1
- data/web/views/queue.erb +10 -1
- data/web/views/retries.erb +4 -1
- data/web/views/retry.erb +2 -2
- data/web/views/scheduled.erb +4 -1
- metadata +4 -2
data/lib/sidekiq/cli.rb
CHANGED
@@ -43,7 +43,7 @@ module Sidekiq
|
|
43
43
|
sigs = %w[INT TERM TTIN TSTP]
|
44
44
|
sigs.each do |sig|
|
45
45
|
trap sig do
|
46
|
-
self_write.
|
46
|
+
self_write.puts(sig)
|
47
47
|
end
|
48
48
|
rescue ArgumentError
|
49
49
|
puts "Signal #{sig} not supported"
|
@@ -162,15 +162,12 @@ module Sidekiq
|
|
162
162
|
end
|
163
163
|
},
|
164
164
|
}
|
165
|
+
UNHANDLED_SIGNAL_HANDLER = ->(cli) { Sidekiq.logger.info "No signal handler registered, ignoring" }
|
166
|
+
SIGNAL_HANDLERS.default = UNHANDLED_SIGNAL_HANDLER
|
165
167
|
|
166
168
|
def handle_signal(sig)
|
167
169
|
Sidekiq.logger.debug "Got #{sig} signal"
|
168
|
-
|
169
|
-
if handy
|
170
|
-
handy.call(self)
|
171
|
-
else
|
172
|
-
Sidekiq.logger.info { "No signal handler for #{sig}" }
|
173
|
-
end
|
170
|
+
SIGNAL_HANDLERS[sig].call(self)
|
174
171
|
end
|
175
172
|
|
176
173
|
private
|
@@ -204,7 +201,7 @@ module Sidekiq
|
|
204
201
|
|
205
202
|
# check config file presence
|
206
203
|
if opts[:config_file]
|
207
|
-
|
204
|
+
unless File.exist?(opts[:config_file])
|
208
205
|
raise ArgumentError, "No such file #{opts[:config_file]}"
|
209
206
|
end
|
210
207
|
else
|
@@ -224,7 +221,7 @@ module Sidekiq
|
|
224
221
|
opts = parse_config(opts[:config_file]).merge(opts) if opts[:config_file]
|
225
222
|
|
226
223
|
# set defaults
|
227
|
-
opts[:queues] =
|
224
|
+
opts[:queues] = ["default"] if opts[:queues].nil? || opts[:queues].empty?
|
228
225
|
opts[:strict] = true if opts[:strict].nil?
|
229
226
|
opts[:concurrency] = Integer(ENV["RAILS_MAX_THREADS"]) if opts[:concurrency].nil? && ENV["RAILS_MAX_THREADS"]
|
230
227
|
|
@@ -283,8 +280,13 @@ module Sidekiq
|
|
283
280
|
|
284
281
|
def parse_options(argv)
|
285
282
|
opts = {}
|
283
|
+
@parser = option_parser(opts)
|
284
|
+
@parser.parse!(argv)
|
285
|
+
opts
|
286
|
+
end
|
286
287
|
|
287
|
-
|
288
|
+
def option_parser(opts)
|
289
|
+
parser = OptionParser.new { |o|
|
288
290
|
o.on "-c", "--concurrency INT", "processor threads to use" do |arg|
|
289
291
|
opts[:concurrency] = Integer(arg)
|
290
292
|
end
|
@@ -336,15 +338,13 @@ module Sidekiq
|
|
336
338
|
end
|
337
339
|
}
|
338
340
|
|
339
|
-
|
340
|
-
|
341
|
-
logger.info
|
341
|
+
parser.banner = "sidekiq [options]"
|
342
|
+
parser.on_tail "-h", "--help", "Show help" do
|
343
|
+
logger.info parser
|
342
344
|
die 1
|
343
345
|
end
|
344
346
|
|
345
|
-
|
346
|
-
|
347
|
-
opts
|
347
|
+
parser
|
348
348
|
end
|
349
349
|
|
350
350
|
def initialize_logger
|
data/lib/sidekiq/client.rb
CHANGED
@@ -94,9 +94,14 @@ module Sidekiq
|
|
94
94
|
return [] unless arg # no jobs to push
|
95
95
|
raise ArgumentError, "Bulk arguments must be an Array of Arrays: [[1], [2]]" unless arg.is_a?(Array)
|
96
96
|
|
97
|
+
at = items.delete("at")
|
98
|
+
raise ArgumentError, "Job 'at' must be a Numeric or an Array of Numeric timestamps" if at && (Array(at).empty? || !Array(at).all?(Numeric))
|
99
|
+
|
97
100
|
normed = normalize_item(items)
|
98
|
-
payloads = items["args"].map { |args|
|
99
|
-
|
101
|
+
payloads = items["args"].map.with_index { |args, index|
|
102
|
+
single_at = at.is_a?(Array) ? at[index] : at
|
103
|
+
copy = normed.merge("args" => args, "jid" => SecureRandom.hex(12), "at" => single_at, "enqueued_at" => Time.now.to_f)
|
104
|
+
|
100
105
|
result = process_single(items["class"], copy)
|
101
106
|
result || nil
|
102
107
|
}.compact
|
@@ -218,6 +223,7 @@ module Sidekiq
|
|
218
223
|
raise(ArgumentError, "Job args must be an Array") unless item["args"].is_a?(Array)
|
219
224
|
raise(ArgumentError, "Job class must be either a Class or String representation of the class name") unless item["class"].is_a?(Class) || item["class"].is_a?(String)
|
220
225
|
raise(ArgumentError, "Job 'at' must be a Numeric timestamp") if item.key?("at") && !item["at"].is_a?(Numeric)
|
226
|
+
raise(ArgumentError, "Job tags must be an Array") if item["tags"] && !item["tags"].is_a?(Array)
|
221
227
|
# raise(ArgumentError, "Arguments must be native JSON types, see https://github.com/mperham/sidekiq/wiki/Best-Practices") unless JSON.load(JSON.dump(item['args'])) == item['args']
|
222
228
|
|
223
229
|
normalized_hash(item["class"])
|
data/lib/sidekiq/fetch.rb
CHANGED
@@ -14,12 +14,12 @@ module Sidekiq
|
|
14
14
|
end
|
15
15
|
|
16
16
|
def queue_name
|
17
|
-
queue.
|
17
|
+
queue.delete_prefix("queue:")
|
18
18
|
end
|
19
19
|
|
20
20
|
def requeue
|
21
21
|
Sidekiq.redis do |conn|
|
22
|
-
conn.rpush(
|
22
|
+
conn.rpush(queue, job)
|
23
23
|
end
|
24
24
|
end
|
25
25
|
}
|
@@ -28,7 +28,7 @@ module Sidekiq
|
|
28
28
|
@strictly_ordered_queues = !!options[:strict]
|
29
29
|
@queues = options[:queues].map { |q| "queue:#{q}" }
|
30
30
|
if @strictly_ordered_queues
|
31
|
-
@queues
|
31
|
+
@queues.uniq!
|
32
32
|
@queues << TIMEOUT
|
33
33
|
end
|
34
34
|
end
|
@@ -47,7 +47,7 @@ module Sidekiq
|
|
47
47
|
if @strictly_ordered_queues
|
48
48
|
@queues
|
49
49
|
else
|
50
|
-
queues = @queues.shuffle
|
50
|
+
queues = @queues.shuffle!.uniq
|
51
51
|
queues << TIMEOUT
|
52
52
|
queues
|
53
53
|
end
|
@@ -61,14 +61,14 @@ module Sidekiq
|
|
61
61
|
Sidekiq.logger.debug { "Re-queueing terminated jobs" }
|
62
62
|
jobs_to_requeue = {}
|
63
63
|
inprogress.each do |unit_of_work|
|
64
|
-
jobs_to_requeue[unit_of_work.
|
65
|
-
jobs_to_requeue[unit_of_work.
|
64
|
+
jobs_to_requeue[unit_of_work.queue] ||= []
|
65
|
+
jobs_to_requeue[unit_of_work.queue] << unit_of_work.job
|
66
66
|
end
|
67
67
|
|
68
68
|
Sidekiq.redis do |conn|
|
69
69
|
conn.pipelined do
|
70
70
|
jobs_to_requeue.each do |queue, jobs|
|
71
|
-
conn.rpush(
|
71
|
+
conn.rpush(queue, jobs)
|
72
72
|
end
|
73
73
|
end
|
74
74
|
end
|
data/lib/sidekiq/job_logger.rb
CHANGED
@@ -23,8 +23,15 @@ module Sidekiq
|
|
23
23
|
raise
|
24
24
|
end
|
25
25
|
|
26
|
-
def
|
27
|
-
|
26
|
+
def prepare(job_hash, &block)
|
27
|
+
level = job_hash["log_level"]
|
28
|
+
if level
|
29
|
+
@logger.log_at(level) do
|
30
|
+
Sidekiq::Context.with(job_hash_context(job_hash), &block)
|
31
|
+
end
|
32
|
+
else
|
33
|
+
Sidekiq::Context.with(job_hash_context(job_hash), &block)
|
34
|
+
end
|
28
35
|
end
|
29
36
|
|
30
37
|
def job_hash_context(job_hash)
|
@@ -35,11 +42,12 @@ module Sidekiq
|
|
35
42
|
jid: job_hash["jid"],
|
36
43
|
}
|
37
44
|
h[:bid] = job_hash["bid"] if job_hash["bid"]
|
45
|
+
h[:tags] = job_hash["tags"] if job_hash["tags"]
|
38
46
|
h
|
39
47
|
end
|
40
48
|
|
41
49
|
def with_elapsed_time_context(start, &block)
|
42
|
-
|
50
|
+
Sidekiq::Context.with(elapsed_time_context(start), &block)
|
43
51
|
end
|
44
52
|
|
45
53
|
def elapsed_time_context(start)
|
data/lib/sidekiq/job_retry.rb
CHANGED
@@ -3,6 +3,9 @@
|
|
3
3
|
require "sidekiq/scheduled"
|
4
4
|
require "sidekiq/api"
|
5
5
|
|
6
|
+
require "zlib"
|
7
|
+
require "base64"
|
8
|
+
|
6
9
|
module Sidekiq
|
7
10
|
##
|
8
11
|
# Automatically retry jobs that fail in Sidekiq.
|
@@ -151,12 +154,14 @@ module Sidekiq
|
|
151
154
|
msg["retry_count"] = 0
|
152
155
|
end
|
153
156
|
|
154
|
-
if msg["backtrace"]
|
155
|
-
msg["
|
156
|
-
|
157
|
-
|
158
|
-
|
159
|
-
|
157
|
+
if msg["backtrace"]
|
158
|
+
lines = if msg["backtrace"] == true
|
159
|
+
exception.backtrace
|
160
|
+
else
|
161
|
+
exception.backtrace[0...msg["backtrace"].to_i]
|
162
|
+
end
|
163
|
+
|
164
|
+
msg["error_backtrace"] = compress_backtrace(lines)
|
160
165
|
end
|
161
166
|
|
162
167
|
if count < max_retry_attempts
|
@@ -245,5 +250,11 @@ module Sidekiq
|
|
245
250
|
rescue
|
246
251
|
+"!!! ERROR MESSAGE THREW AN ERROR !!!"
|
247
252
|
end
|
253
|
+
|
254
|
+
def compress_backtrace(backtrace)
|
255
|
+
serialized = Marshal.dump(backtrace)
|
256
|
+
compressed = Zlib::Deflate.deflate(serialized)
|
257
|
+
Base64.encode64(compressed)
|
258
|
+
end
|
248
259
|
end
|
249
260
|
end
|
data/lib/sidekiq/launcher.rb
CHANGED
@@ -129,15 +129,13 @@ module Sidekiq
|
|
129
129
|
fails = procd = 0
|
130
130
|
|
131
131
|
_, exists, _, _, msg = Sidekiq.redis { |conn|
|
132
|
-
|
132
|
+
conn.multi {
|
133
133
|
conn.sadd("processes", key)
|
134
134
|
conn.exists(key)
|
135
135
|
conn.hmset(key, "info", to_json, "busy", curstate.size, "beat", Time.now.to_f, "quiet", @done)
|
136
136
|
conn.expire(key, 60)
|
137
137
|
conn.rpop("#{key}-signals")
|
138
138
|
}
|
139
|
-
|
140
|
-
res
|
141
139
|
}
|
142
140
|
|
143
141
|
# first heartbeat or recovering from an outage and need to reestablish our heartbeat
|
data/lib/sidekiq/logger.rb
CHANGED
@@ -4,22 +4,109 @@ require "logger"
|
|
4
4
|
require "time"
|
5
5
|
|
6
6
|
module Sidekiq
|
7
|
-
|
8
|
-
def
|
9
|
-
|
7
|
+
module Context
|
8
|
+
def self.with(hash)
|
9
|
+
current.merge!(hash)
|
10
|
+
yield
|
11
|
+
ensure
|
12
|
+
hash.each_key { |key| current.delete(key) }
|
13
|
+
end
|
10
14
|
|
11
|
-
|
15
|
+
def self.current
|
16
|
+
Thread.current[:sidekiq_context] ||= {}
|
17
|
+
end
|
18
|
+
end
|
19
|
+
|
20
|
+
module LoggingUtils
|
21
|
+
LEVELS = {
|
22
|
+
"debug" => 0,
|
23
|
+
"info" => 1,
|
24
|
+
"warn" => 2,
|
25
|
+
"error" => 3,
|
26
|
+
"fatal" => 4,
|
27
|
+
}
|
28
|
+
LEVELS.default_proc = proc do |_, level|
|
29
|
+
Sidekiq.logger.warn("Invalid log level: #{level.inspect}")
|
30
|
+
nil
|
31
|
+
end
|
32
|
+
|
33
|
+
def debug?
|
34
|
+
level >= 0
|
35
|
+
end
|
36
|
+
|
37
|
+
def info?
|
38
|
+
level >= 1
|
39
|
+
end
|
40
|
+
|
41
|
+
def warn?
|
42
|
+
level >= 2
|
43
|
+
end
|
44
|
+
|
45
|
+
def error?
|
46
|
+
level >= 3
|
47
|
+
end
|
48
|
+
|
49
|
+
def fatal?
|
50
|
+
level >= 4
|
12
51
|
end
|
13
52
|
|
14
|
-
def
|
15
|
-
|
53
|
+
def local_level
|
54
|
+
Thread.current[:sidekiq_log_level]
|
55
|
+
end
|
56
|
+
|
57
|
+
def local_level=(level)
|
58
|
+
case level
|
59
|
+
when Integer
|
60
|
+
Thread.current[:sidekiq_log_level] = level
|
61
|
+
when Symbol, String
|
62
|
+
Thread.current[:sidekiq_log_level] = LEVELS[level.to_s]
|
63
|
+
when nil
|
64
|
+
Thread.current[:sidekiq_log_level] = nil
|
65
|
+
else
|
66
|
+
raise ArgumentError, "Invalid log level: #{level.inspect}"
|
67
|
+
end
|
68
|
+
end
|
69
|
+
|
70
|
+
def level
|
71
|
+
local_level || super
|
72
|
+
end
|
73
|
+
|
74
|
+
# Change the thread-local level for the duration of the given block.
|
75
|
+
def log_at(level)
|
76
|
+
old_local_level = local_level
|
77
|
+
self.local_level = level
|
16
78
|
yield
|
17
79
|
ensure
|
18
|
-
|
80
|
+
self.local_level = old_local_level
|
19
81
|
end
|
20
82
|
|
21
|
-
|
22
|
-
|
83
|
+
# Redefined to check severity against #level, and thus the thread-local level, rather than +@level+.
|
84
|
+
# FIXME: Remove when the minimum Ruby version supports overriding Logger#level.
|
85
|
+
def add(severity, message = nil, progname = nil, &block)
|
86
|
+
severity ||= UNKNOWN
|
87
|
+
progname ||= @progname
|
88
|
+
|
89
|
+
return true if @logdev.nil? || severity < level
|
90
|
+
|
91
|
+
if message.nil?
|
92
|
+
if block_given?
|
93
|
+
message = yield
|
94
|
+
else
|
95
|
+
message = progname
|
96
|
+
progname = @progname
|
97
|
+
end
|
98
|
+
end
|
99
|
+
|
100
|
+
@logdev.write format_message(format_severity(severity), Time.now, progname, message)
|
101
|
+
end
|
102
|
+
end
|
103
|
+
|
104
|
+
class Logger < ::Logger
|
105
|
+
include LoggingUtils
|
106
|
+
|
107
|
+
def initialize(*args)
|
108
|
+
super
|
109
|
+
self.formatter = Sidekiq.log_formatter
|
23
110
|
end
|
24
111
|
|
25
112
|
module Formatters
|
@@ -29,11 +116,20 @@ module Sidekiq
|
|
29
116
|
end
|
30
117
|
|
31
118
|
def ctx
|
32
|
-
|
119
|
+
Sidekiq::Context.current
|
33
120
|
end
|
34
121
|
|
35
122
|
def format_context
|
36
|
-
|
123
|
+
if ctx.any?
|
124
|
+
" " + ctx.compact.map { |k, v|
|
125
|
+
case v
|
126
|
+
when Array
|
127
|
+
"#{k}=#{v.join(",")}"
|
128
|
+
else
|
129
|
+
"#{k}=#{v}"
|
130
|
+
end
|
131
|
+
}.join(" ")
|
132
|
+
end
|
37
133
|
end
|
38
134
|
end
|
39
135
|
|
@@ -67,7 +67,6 @@ module Sidekiq
|
|
67
67
|
module Middleware
|
68
68
|
class Chain
|
69
69
|
include Enumerable
|
70
|
-
attr_reader :entries
|
71
70
|
|
72
71
|
def initialize_copy(copy)
|
73
72
|
copy.instance_variable_set(:@entries, entries.dup)
|
@@ -78,10 +77,14 @@ module Sidekiq
|
|
78
77
|
end
|
79
78
|
|
80
79
|
def initialize
|
81
|
-
@entries =
|
80
|
+
@entries = nil
|
82
81
|
yield self if block_given?
|
83
82
|
end
|
84
83
|
|
84
|
+
def entries
|
85
|
+
@entries ||= []
|
86
|
+
end
|
87
|
+
|
85
88
|
def remove(klass)
|
86
89
|
entries.delete_if { |entry| entry.klass == klass }
|
87
90
|
end
|
@@ -114,6 +117,10 @@ module Sidekiq
|
|
114
117
|
any? { |entry| entry.klass == klass }
|
115
118
|
end
|
116
119
|
|
120
|
+
def empty?
|
121
|
+
@entries.nil? || @entries.empty?
|
122
|
+
end
|
123
|
+
|
117
124
|
def retrieve
|
118
125
|
map(&:make_new)
|
119
126
|
end
|
@@ -123,6 +130,8 @@ module Sidekiq
|
|
123
130
|
end
|
124
131
|
|
125
132
|
def invoke(*args)
|
133
|
+
return yield if empty?
|
134
|
+
|
126
135
|
chain = retrieve.dup
|
127
136
|
traverse_chain = lambda do
|
128
137
|
if chain.empty?
|
data/lib/sidekiq/monitor.rb
CHANGED
@@ -11,7 +11,7 @@ class Sidekiq::Monitor
|
|
11
11
|
def self.print_usage
|
12
12
|
puts "#{CMD} - monitor Sidekiq from the command line."
|
13
13
|
puts
|
14
|
-
puts "Usage: #{CMD}
|
14
|
+
puts "Usage: #{CMD} <section>"
|
15
15
|
puts
|
16
16
|
puts " <section> (optional) view a specific section of the status output"
|
17
17
|
puts " Valid sections are: #{Sidekiq::Monitor::Status::VALID_SECTIONS.join(", ")}"
|
@@ -47,7 +47,7 @@ class Sidekiq::Monitor
|
|
47
47
|
|
48
48
|
def version
|
49
49
|
puts "Sidekiq #{Sidekiq::VERSION}"
|
50
|
-
puts Time.now
|
50
|
+
puts Time.now.utc
|
51
51
|
end
|
52
52
|
|
53
53
|
def overview
|