sidekiq 6.0.0 → 6.0.6
Sign up to get free protection for your applications and to get access to all the features.
Potentially problematic release.
This version of sidekiq might be problematic. Click here for more details.
- checksums.yaml +4 -4
- data/.circleci/config.yml +0 -1
- data/6.0-Upgrade.md +3 -1
- data/Changes.md +122 -1
- data/Ent-Changes.md +7 -1
- data/Gemfile +1 -1
- data/Gemfile.lock +106 -94
- data/Pro-Changes.md +16 -2
- data/README.md +3 -1
- data/bin/sidekiqload +8 -4
- data/bin/sidekiqmon +4 -5
- data/lib/generators/sidekiq/worker_generator.rb +11 -1
- data/lib/sidekiq/api.rb +125 -92
- data/lib/sidekiq/cli.rb +32 -19
- data/lib/sidekiq/client.rb +20 -4
- data/lib/sidekiq/fetch.rb +7 -7
- data/lib/sidekiq/job_logger.rb +12 -4
- data/lib/sidekiq/job_retry.rb +23 -10
- data/lib/sidekiq/launcher.rb +23 -7
- data/lib/sidekiq/logger.rb +108 -12
- data/lib/sidekiq/middleware/chain.rb +11 -2
- data/lib/sidekiq/monitor.rb +3 -18
- data/lib/sidekiq/paginator.rb +7 -2
- data/lib/sidekiq/processor.rb +18 -20
- data/lib/sidekiq/redis_connection.rb +3 -0
- data/lib/sidekiq/scheduled.rb +13 -12
- data/lib/sidekiq/sd_notify.rb +149 -0
- data/lib/sidekiq/systemd.rb +38 -0
- data/lib/sidekiq/testing.rb +12 -0
- data/lib/sidekiq/util.rb +0 -2
- data/lib/sidekiq/version.rb +1 -1
- data/lib/sidekiq/web/application.rb +22 -21
- data/lib/sidekiq/web/helpers.rb +23 -11
- data/lib/sidekiq/web/router.rb +1 -3
- data/lib/sidekiq/web.rb +1 -1
- data/lib/sidekiq/worker.rb +6 -6
- data/lib/sidekiq.rb +17 -5
- data/sidekiq.gemspec +2 -2
- data/web/assets/javascripts/application.js +22 -19
- data/web/assets/javascripts/dashboard.js +2 -2
- data/web/assets/stylesheets/application-dark.css +122 -0
- data/web/assets/stylesheets/application.css +9 -0
- data/web/locales/de.yml +14 -2
- data/web/locales/en.yml +2 -0
- data/web/locales/ja.yml +2 -0
- data/web/locales/lt.yml +83 -0
- data/web/views/_job_info.erb +2 -1
- data/web/views/busy.erb +4 -1
- data/web/views/dead.erb +2 -2
- data/web/views/layout.erb +1 -0
- data/web/views/morgue.erb +4 -1
- data/web/views/queue.erb +10 -1
- data/web/views/queues.erb +8 -0
- data/web/views/retries.erb +4 -1
- data/web/views/retry.erb +2 -2
- data/web/views/scheduled.erb +4 -1
- metadata +12 -8
data/lib/sidekiq/job_logger.rb
CHANGED
@@ -23,8 +23,15 @@ module Sidekiq
|
|
23
23
|
raise
|
24
24
|
end
|
25
25
|
|
26
|
-
def
|
27
|
-
|
26
|
+
def prepare(job_hash, &block)
|
27
|
+
level = job_hash["log_level"]
|
28
|
+
if level
|
29
|
+
@logger.log_at(level) do
|
30
|
+
Sidekiq::Context.with(job_hash_context(job_hash), &block)
|
31
|
+
end
|
32
|
+
else
|
33
|
+
Sidekiq::Context.with(job_hash_context(job_hash), &block)
|
34
|
+
end
|
28
35
|
end
|
29
36
|
|
30
37
|
def job_hash_context(job_hash)
|
@@ -32,14 +39,15 @@ module Sidekiq
|
|
32
39
|
# attribute to expose the underlying thing.
|
33
40
|
h = {
|
34
41
|
class: job_hash["wrapped"] || job_hash["class"],
|
35
|
-
jid: job_hash["jid"]
|
42
|
+
jid: job_hash["jid"]
|
36
43
|
}
|
37
44
|
h[:bid] = job_hash["bid"] if job_hash["bid"]
|
45
|
+
h[:tags] = job_hash["tags"] if job_hash["tags"]
|
38
46
|
h
|
39
47
|
end
|
40
48
|
|
41
49
|
def with_elapsed_time_context(start, &block)
|
42
|
-
|
50
|
+
Sidekiq::Context.with(elapsed_time_context(start), &block)
|
43
51
|
end
|
44
52
|
|
45
53
|
def elapsed_time_context(start)
|
data/lib/sidekiq/job_retry.rb
CHANGED
@@ -3,6 +3,9 @@
|
|
3
3
|
require "sidekiq/scheduled"
|
4
4
|
require "sidekiq/api"
|
5
5
|
|
6
|
+
require "zlib"
|
7
|
+
require "base64"
|
8
|
+
|
6
9
|
module Sidekiq
|
7
10
|
##
|
8
11
|
# Automatically retry jobs that fail in Sidekiq.
|
@@ -71,7 +74,7 @@ module Sidekiq
|
|
71
74
|
# The global retry handler requires only the barest of data.
|
72
75
|
# We want to be able to retry as much as possible so we don't
|
73
76
|
# require the worker to be instantiated.
|
74
|
-
def global(
|
77
|
+
def global(jobstr, queue)
|
75
78
|
yield
|
76
79
|
rescue Handled => ex
|
77
80
|
raise ex
|
@@ -82,6 +85,7 @@ module Sidekiq
|
|
82
85
|
# ignore, will be pushed back onto queue during hard_shutdown
|
83
86
|
raise Sidekiq::Shutdown if exception_caused_by_shutdown?(e)
|
84
87
|
|
88
|
+
msg = Sidekiq.load_json(jobstr)
|
85
89
|
if msg["retry"]
|
86
90
|
attempt_retry(nil, msg, queue, e)
|
87
91
|
else
|
@@ -103,7 +107,7 @@ module Sidekiq
|
|
103
107
|
# exception so the global block does not reprocess the error. The
|
104
108
|
# Skip exception is unwrapped within Sidekiq::Processor#process before
|
105
109
|
# calling the handle_exception handlers.
|
106
|
-
def local(worker,
|
110
|
+
def local(worker, jobstr, queue)
|
107
111
|
yield
|
108
112
|
rescue Handled => ex
|
109
113
|
raise ex
|
@@ -114,6 +118,7 @@ module Sidekiq
|
|
114
118
|
# ignore, will be pushed back onto queue during hard_shutdown
|
115
119
|
raise Sidekiq::Shutdown if exception_caused_by_shutdown?(e)
|
116
120
|
|
121
|
+
msg = Sidekiq.load_json(jobstr)
|
117
122
|
if msg["retry"].nil?
|
118
123
|
msg["retry"] = worker.class.get_sidekiq_options["retry"]
|
119
124
|
end
|
@@ -151,12 +156,14 @@ module Sidekiq
|
|
151
156
|
msg["retry_count"] = 0
|
152
157
|
end
|
153
158
|
|
154
|
-
if msg["backtrace"]
|
155
|
-
msg["
|
156
|
-
|
157
|
-
|
158
|
-
|
159
|
-
|
159
|
+
if msg["backtrace"]
|
160
|
+
lines = if msg["backtrace"] == true
|
161
|
+
exception.backtrace
|
162
|
+
else
|
163
|
+
exception.backtrace[0...msg["backtrace"].to_i]
|
164
|
+
end
|
165
|
+
|
166
|
+
msg["error_backtrace"] = compress_backtrace(lines)
|
160
167
|
end
|
161
168
|
|
162
169
|
if count < max_retry_attempts
|
@@ -182,13 +189,13 @@ module Sidekiq
|
|
182
189
|
handle_exception(e, {context: "Error calling retries_exhausted", job: msg})
|
183
190
|
end
|
184
191
|
|
192
|
+
send_to_morgue(msg) unless msg["dead"] == false
|
193
|
+
|
185
194
|
Sidekiq.death_handlers.each do |handler|
|
186
195
|
handler.call(msg, exception)
|
187
196
|
rescue => e
|
188
197
|
handle_exception(e, {context: "Error calling death handler", job: msg})
|
189
198
|
end
|
190
|
-
|
191
|
-
send_to_morgue(msg) unless msg["dead"] == false
|
192
199
|
end
|
193
200
|
|
194
201
|
def send_to_morgue(msg)
|
@@ -245,5 +252,11 @@ module Sidekiq
|
|
245
252
|
rescue
|
246
253
|
+"!!! ERROR MESSAGE THREW AN ERROR !!!"
|
247
254
|
end
|
255
|
+
|
256
|
+
def compress_backtrace(backtrace)
|
257
|
+
serialized = Sidekiq.dump_json(backtrace)
|
258
|
+
compressed = Zlib::Deflate.deflate(serialized)
|
259
|
+
Base64.encode64(compressed)
|
260
|
+
end
|
248
261
|
end
|
249
262
|
end
|
data/lib/sidekiq/launcher.rb
CHANGED
@@ -16,7 +16,7 @@ module Sidekiq
|
|
16
16
|
proc { Sidekiq::VERSION },
|
17
17
|
proc { |me, data| data["tag"] },
|
18
18
|
proc { |me, data| "[#{Processor::WORKER_STATE.size} of #{data["concurrency"]} busy]" },
|
19
|
-
proc { |me, data| "stopping" if me.stopping? }
|
19
|
+
proc { |me, data| "stopping" if me.stopping? }
|
20
20
|
]
|
21
21
|
|
22
22
|
attr_accessor :manager, :poller, :fetcher
|
@@ -83,7 +83,7 @@ module Sidekiq
|
|
83
83
|
Sidekiq.redis do |conn|
|
84
84
|
conn.pipelined do
|
85
85
|
conn.srem("processes", identity)
|
86
|
-
conn.
|
86
|
+
conn.unlink("#{identity}:workers")
|
87
87
|
end
|
88
88
|
end
|
89
89
|
rescue
|
@@ -96,6 +96,24 @@ module Sidekiq
|
|
96
96
|
❤
|
97
97
|
end
|
98
98
|
|
99
|
+
def self.flush_stats
|
100
|
+
nowdate = Time.now.utc.strftime("%Y-%m-%d")
|
101
|
+
fails = Processor::FAILURE.reset
|
102
|
+
procd = Processor::PROCESSED.reset
|
103
|
+
Sidekiq.redis do |conn|
|
104
|
+
conn.pipelined do
|
105
|
+
conn.incrby("stat:processed", procd)
|
106
|
+
conn.incrby("stat:processed:#{nowdate}", procd)
|
107
|
+
conn.expire("stat:processed:#{nowdate}", STATS_TTL)
|
108
|
+
|
109
|
+
conn.incrby("stat:failed", fails)
|
110
|
+
conn.incrby("stat:failed:#{nowdate}", fails)
|
111
|
+
conn.expire("stat:failed:#{nowdate}", STATS_TTL)
|
112
|
+
end
|
113
|
+
end
|
114
|
+
end
|
115
|
+
at_exit(&method(:flush_stats))
|
116
|
+
|
99
117
|
def ❤
|
100
118
|
key = identity
|
101
119
|
fails = procd = 0
|
@@ -118,7 +136,7 @@ module Sidekiq
|
|
118
136
|
conn.incrby("stat:failed:#{nowdate}", fails)
|
119
137
|
conn.expire("stat:failed:#{nowdate}", STATS_TTL)
|
120
138
|
|
121
|
-
conn.
|
139
|
+
conn.unlink(workers_key)
|
122
140
|
curstate.each_pair do |tid, hash|
|
123
141
|
conn.hset(workers_key, tid, Sidekiq.dump_json(hash))
|
124
142
|
end
|
@@ -129,15 +147,13 @@ module Sidekiq
|
|
129
147
|
fails = procd = 0
|
130
148
|
|
131
149
|
_, exists, _, _, msg = Sidekiq.redis { |conn|
|
132
|
-
|
150
|
+
conn.multi {
|
133
151
|
conn.sadd("processes", key)
|
134
152
|
conn.exists(key)
|
135
153
|
conn.hmset(key, "info", to_json, "busy", curstate.size, "beat", Time.now.to_f, "quiet", @done)
|
136
154
|
conn.expire(key, 60)
|
137
155
|
conn.rpop("#{key}-signals")
|
138
156
|
}
|
139
|
-
|
140
|
-
res
|
141
157
|
}
|
142
158
|
|
143
159
|
# first heartbeat or recovering from an outage and need to reestablish our heartbeat
|
@@ -165,7 +181,7 @@ module Sidekiq
|
|
165
181
|
"concurrency" => @options[:concurrency],
|
166
182
|
"queues" => @options[:queues].uniq,
|
167
183
|
"labels" => @options[:labels],
|
168
|
-
"identity" => identity
|
184
|
+
"identity" => identity
|
169
185
|
}
|
170
186
|
end
|
171
187
|
end
|
data/lib/sidekiq/logger.rb
CHANGED
@@ -4,22 +4,109 @@ require "logger"
|
|
4
4
|
require "time"
|
5
5
|
|
6
6
|
module Sidekiq
|
7
|
-
|
8
|
-
def
|
9
|
-
|
7
|
+
module Context
|
8
|
+
def self.with(hash)
|
9
|
+
current.merge!(hash)
|
10
|
+
yield
|
11
|
+
ensure
|
12
|
+
hash.each_key { |key| current.delete(key) }
|
13
|
+
end
|
10
14
|
|
11
|
-
|
15
|
+
def self.current
|
16
|
+
Thread.current[:sidekiq_context] ||= {}
|
17
|
+
end
|
18
|
+
end
|
19
|
+
|
20
|
+
module LoggingUtils
|
21
|
+
LEVELS = {
|
22
|
+
"debug" => 0,
|
23
|
+
"info" => 1,
|
24
|
+
"warn" => 2,
|
25
|
+
"error" => 3,
|
26
|
+
"fatal" => 4
|
27
|
+
}
|
28
|
+
LEVELS.default_proc = proc do |_, level|
|
29
|
+
Sidekiq.logger.warn("Invalid log level: #{level.inspect}")
|
30
|
+
nil
|
31
|
+
end
|
32
|
+
|
33
|
+
def debug?
|
34
|
+
level <= 0
|
35
|
+
end
|
36
|
+
|
37
|
+
def info?
|
38
|
+
level <= 1
|
39
|
+
end
|
40
|
+
|
41
|
+
def warn?
|
42
|
+
level <= 2
|
43
|
+
end
|
44
|
+
|
45
|
+
def error?
|
46
|
+
level <= 3
|
47
|
+
end
|
48
|
+
|
49
|
+
def fatal?
|
50
|
+
level <= 4
|
12
51
|
end
|
13
52
|
|
14
|
-
def
|
15
|
-
|
53
|
+
def local_level
|
54
|
+
Thread.current[:sidekiq_log_level]
|
55
|
+
end
|
56
|
+
|
57
|
+
def local_level=(level)
|
58
|
+
case level
|
59
|
+
when Integer
|
60
|
+
Thread.current[:sidekiq_log_level] = level
|
61
|
+
when Symbol, String
|
62
|
+
Thread.current[:sidekiq_log_level] = LEVELS[level.to_s]
|
63
|
+
when nil
|
64
|
+
Thread.current[:sidekiq_log_level] = nil
|
65
|
+
else
|
66
|
+
raise ArgumentError, "Invalid log level: #{level.inspect}"
|
67
|
+
end
|
68
|
+
end
|
69
|
+
|
70
|
+
def level
|
71
|
+
local_level || super
|
72
|
+
end
|
73
|
+
|
74
|
+
# Change the thread-local level for the duration of the given block.
|
75
|
+
def log_at(level)
|
76
|
+
old_local_level = local_level
|
77
|
+
self.local_level = level
|
16
78
|
yield
|
17
79
|
ensure
|
18
|
-
|
80
|
+
self.local_level = old_local_level
|
19
81
|
end
|
20
82
|
|
21
|
-
|
22
|
-
|
83
|
+
# Redefined to check severity against #level, and thus the thread-local level, rather than +@level+.
|
84
|
+
# FIXME: Remove when the minimum Ruby version supports overriding Logger#level.
|
85
|
+
def add(severity, message = nil, progname = nil, &block)
|
86
|
+
severity ||= ::Logger::UNKNOWN
|
87
|
+
progname ||= @progname
|
88
|
+
|
89
|
+
return true if @logdev.nil? || severity < level
|
90
|
+
|
91
|
+
if message.nil?
|
92
|
+
if block_given?
|
93
|
+
message = yield
|
94
|
+
else
|
95
|
+
message = progname
|
96
|
+
progname = @progname
|
97
|
+
end
|
98
|
+
end
|
99
|
+
|
100
|
+
@logdev.write format_message(format_severity(severity), Time.now, progname, message)
|
101
|
+
end
|
102
|
+
end
|
103
|
+
|
104
|
+
class Logger < ::Logger
|
105
|
+
include LoggingUtils
|
106
|
+
|
107
|
+
def initialize(*args, **kwargs)
|
108
|
+
super
|
109
|
+
self.formatter = Sidekiq.log_formatter
|
23
110
|
end
|
24
111
|
|
25
112
|
module Formatters
|
@@ -29,11 +116,20 @@ module Sidekiq
|
|
29
116
|
end
|
30
117
|
|
31
118
|
def ctx
|
32
|
-
|
119
|
+
Sidekiq::Context.current
|
33
120
|
end
|
34
121
|
|
35
122
|
def format_context
|
36
|
-
|
123
|
+
if ctx.any?
|
124
|
+
" " + ctx.compact.map { |k, v|
|
125
|
+
case v
|
126
|
+
when Array
|
127
|
+
"#{k}=#{v.join(",")}"
|
128
|
+
else
|
129
|
+
"#{k}=#{v}"
|
130
|
+
end
|
131
|
+
}.join(" ")
|
132
|
+
end
|
37
133
|
end
|
38
134
|
end
|
39
135
|
|
@@ -56,7 +152,7 @@ module Sidekiq
|
|
56
152
|
pid: ::Process.pid,
|
57
153
|
tid: tid,
|
58
154
|
lvl: severity,
|
59
|
-
msg: message
|
155
|
+
msg: message
|
60
156
|
}
|
61
157
|
c = ctx
|
62
158
|
hash["ctx"] = c unless c.empty?
|
@@ -67,7 +67,6 @@ module Sidekiq
|
|
67
67
|
module Middleware
|
68
68
|
class Chain
|
69
69
|
include Enumerable
|
70
|
-
attr_reader :entries
|
71
70
|
|
72
71
|
def initialize_copy(copy)
|
73
72
|
copy.instance_variable_set(:@entries, entries.dup)
|
@@ -78,10 +77,14 @@ module Sidekiq
|
|
78
77
|
end
|
79
78
|
|
80
79
|
def initialize
|
81
|
-
@entries =
|
80
|
+
@entries = nil
|
82
81
|
yield self if block_given?
|
83
82
|
end
|
84
83
|
|
84
|
+
def entries
|
85
|
+
@entries ||= []
|
86
|
+
end
|
87
|
+
|
85
88
|
def remove(klass)
|
86
89
|
entries.delete_if { |entry| entry.klass == klass }
|
87
90
|
end
|
@@ -114,6 +117,10 @@ module Sidekiq
|
|
114
117
|
any? { |entry| entry.klass == klass }
|
115
118
|
end
|
116
119
|
|
120
|
+
def empty?
|
121
|
+
@entries.nil? || @entries.empty?
|
122
|
+
end
|
123
|
+
|
117
124
|
def retrieve
|
118
125
|
map(&:make_new)
|
119
126
|
end
|
@@ -123,6 +130,8 @@ module Sidekiq
|
|
123
130
|
end
|
124
131
|
|
125
132
|
def invoke(*args)
|
133
|
+
return yield if empty?
|
134
|
+
|
126
135
|
chain = retrieve.dup
|
127
136
|
traverse_chain = lambda do
|
128
137
|
if chain.empty?
|
data/lib/sidekiq/monitor.rb
CHANGED
@@ -4,21 +4,6 @@ require "fileutils"
|
|
4
4
|
require "sidekiq/api"
|
5
5
|
|
6
6
|
class Sidekiq::Monitor
|
7
|
-
CMD = File.basename($PROGRAM_NAME)
|
8
|
-
|
9
|
-
attr_reader :stage
|
10
|
-
|
11
|
-
def self.print_usage
|
12
|
-
puts "#{CMD} - monitor Sidekiq from the command line."
|
13
|
-
puts
|
14
|
-
puts "Usage: #{CMD} status <section>"
|
15
|
-
puts
|
16
|
-
puts " <section> (optional) view a specific section of the status output"
|
17
|
-
puts " Valid sections are: #{Sidekiq::Monitor::Status::VALID_SECTIONS.join(", ")}"
|
18
|
-
puts
|
19
|
-
puts "Set REDIS_URL to the location of your Redis server if not monitoring localhost."
|
20
|
-
end
|
21
|
-
|
22
7
|
class Status
|
23
8
|
VALID_SECTIONS = %w[all version overview processes queues]
|
24
9
|
COL_PAD = 2
|
@@ -47,7 +32,7 @@ class Sidekiq::Monitor
|
|
47
32
|
|
48
33
|
def version
|
49
34
|
puts "Sidekiq #{Sidekiq::VERSION}"
|
50
|
-
puts Time.now
|
35
|
+
puts Time.now.utc
|
51
36
|
end
|
52
37
|
|
53
38
|
def overview
|
@@ -77,7 +62,7 @@ class Sidekiq::Monitor
|
|
77
62
|
columns = {
|
78
63
|
name: [:ljust, (["name"] + queue_data.map(&:name)).map(&:length).max + COL_PAD],
|
79
64
|
size: [:rjust, (["size"] + queue_data.map(&:size)).map(&:length).max + COL_PAD],
|
80
|
-
latency: [:rjust, (["latency"] + queue_data.map(&:latency)).map(&:length).max + COL_PAD]
|
65
|
+
latency: [:rjust, (["latency"] + queue_data.map(&:latency)).map(&:length).max + COL_PAD]
|
81
66
|
}
|
82
67
|
columns.each { |col, (dir, width)| print col.to_s.upcase.public_send(dir, width) }
|
83
68
|
puts
|
@@ -116,7 +101,7 @@ class Sidekiq::Monitor
|
|
116
101
|
tags = [
|
117
102
|
process["tag"],
|
118
103
|
process["labels"],
|
119
|
-
(process["quiet"] == "true" ? "quiet" : nil)
|
104
|
+
(process["quiet"] == "true" ? "quiet" : nil)
|
120
105
|
].flatten.compact
|
121
106
|
tags.any? ? "[#{tags.join("] [")}]" : nil
|
122
107
|
end
|
data/lib/sidekiq/paginator.rb
CHANGED
@@ -12,10 +12,10 @@ module Sidekiq
|
|
12
12
|
|
13
13
|
Sidekiq.redis do |conn|
|
14
14
|
type = conn.type(key)
|
15
|
+
rev = opts && opts[:reverse]
|
15
16
|
|
16
17
|
case type
|
17
18
|
when "zset"
|
18
|
-
rev = opts && opts[:reverse]
|
19
19
|
total_size, items = conn.multi {
|
20
20
|
conn.zcard(key)
|
21
21
|
if rev
|
@@ -28,8 +28,13 @@ module Sidekiq
|
|
28
28
|
when "list"
|
29
29
|
total_size, items = conn.multi {
|
30
30
|
conn.llen(key)
|
31
|
-
|
31
|
+
if rev
|
32
|
+
conn.lrange(key, -ending - 1, -starting - 1)
|
33
|
+
else
|
34
|
+
conn.lrange(key, starting, ending)
|
35
|
+
end
|
32
36
|
}
|
37
|
+
items.reverse! if rev
|
33
38
|
[current_page, total_size, items]
|
34
39
|
when "none"
|
35
40
|
[1, 0, []]
|
data/lib/sidekiq/processor.rb
CHANGED
@@ -111,16 +111,19 @@ module Sidekiq
|
|
111
111
|
nil
|
112
112
|
end
|
113
113
|
|
114
|
-
def dispatch(job_hash, queue)
|
114
|
+
def dispatch(job_hash, queue, jobstr)
|
115
115
|
# since middleware can mutate the job hash
|
116
|
-
# we clone
|
116
|
+
# we need to clone it to report the original
|
117
117
|
# job structure to the Web UI
|
118
|
-
|
118
|
+
# or to push back to redis when retrying.
|
119
|
+
# To avoid costly and, most of the time, useless cloning here,
|
120
|
+
# we pass original String of JSON to respected methods
|
121
|
+
# to re-parse it there if we need access to the original, untouched job
|
119
122
|
|
120
|
-
@job_logger.
|
121
|
-
@retrier.global(
|
123
|
+
@job_logger.prepare(job_hash) do
|
124
|
+
@retrier.global(jobstr, queue) do
|
122
125
|
@job_logger.call(job_hash, queue) do
|
123
|
-
stats(
|
126
|
+
stats(jobstr, queue) do
|
124
127
|
# Rails 5 requires a Reloader to wrap code execution. In order to
|
125
128
|
# constantize the worker and instantiate an instance, we have to call
|
126
129
|
# the Reloader. It handles code loading, db connection management, etc.
|
@@ -129,7 +132,7 @@ module Sidekiq
|
|
129
132
|
klass = constantize(job_hash["class"])
|
130
133
|
worker = klass.new
|
131
134
|
worker.jid = job_hash["jid"]
|
132
|
-
@retrier.local(worker,
|
135
|
+
@retrier.local(worker, jobstr, queue) do
|
133
136
|
yield worker
|
134
137
|
end
|
135
138
|
end
|
@@ -156,9 +159,9 @@ module Sidekiq
|
|
156
159
|
|
157
160
|
ack = false
|
158
161
|
begin
|
159
|
-
dispatch(job_hash, queue) do |worker|
|
162
|
+
dispatch(job_hash, queue, jobstr) do |worker|
|
160
163
|
Sidekiq.server_middleware.invoke(worker, job_hash, queue) do
|
161
|
-
execute_job(worker,
|
164
|
+
execute_job(worker, job_hash["args"])
|
162
165
|
end
|
163
166
|
end
|
164
167
|
ack = true
|
@@ -178,7 +181,7 @@ module Sidekiq
|
|
178
181
|
# the retry subsystem (e.g. network partition). We won't acknowledge the job
|
179
182
|
# so it can be rescued when using Sidekiq Pro.
|
180
183
|
handle_exception(ex, {context: "Internal exception!", job: job_hash, jobstr: jobstr})
|
181
|
-
raise
|
184
|
+
raise ex
|
182
185
|
ensure
|
183
186
|
if ack
|
184
187
|
# We don't want a shutdown signal to interrupt job acknowledgment.
|
@@ -247,8 +250,8 @@ module Sidekiq
|
|
247
250
|
FAILURE = Counter.new
|
248
251
|
WORKER_STATE = SharedWorkerState.new
|
249
252
|
|
250
|
-
def stats(
|
251
|
-
WORKER_STATE.set(tid, {queue: queue, payload:
|
253
|
+
def stats(jobstr, queue)
|
254
|
+
WORKER_STATE.set(tid, {queue: queue, payload: jobstr, run_at: Time.now.to_i})
|
252
255
|
|
253
256
|
begin
|
254
257
|
yield
|
@@ -261,21 +264,16 @@ module Sidekiq
|
|
261
264
|
end
|
262
265
|
end
|
263
266
|
|
264
|
-
# Deep clone the arguments passed to the worker so that if
|
265
|
-
# the job fails, what is pushed back onto Redis hasn't
|
266
|
-
# been mutated by the worker.
|
267
|
-
def cloned(thing)
|
268
|
-
Marshal.load(Marshal.dump(thing))
|
269
|
-
end
|
270
|
-
|
271
267
|
def constantize(str)
|
268
|
+
return Object.const_get(str) unless str.include?("::")
|
269
|
+
|
272
270
|
names = str.split("::")
|
273
271
|
names.shift if names.empty? || names.first.empty?
|
274
272
|
|
275
273
|
names.inject(Object) do |constant, name|
|
276
274
|
# the false flag limits search for name to under the constant namespace
|
277
275
|
# which mimics Rails' behaviour
|
278
|
-
constant.
|
276
|
+
constant.const_get(name, false)
|
279
277
|
end
|
280
278
|
end
|
281
279
|
end
|
@@ -103,6 +103,9 @@ module Sidekiq
|
|
103
103
|
if scrubbed_options[:password]
|
104
104
|
scrubbed_options[:password] = redacted
|
105
105
|
end
|
106
|
+
scrubbed_options[:sentinels]&.each do |sentinel|
|
107
|
+
sentinel[:password] = redacted if sentinel[:password]
|
108
|
+
end
|
106
109
|
if Sidekiq.server?
|
107
110
|
Sidekiq.logger.info("Booting Sidekiq #{Sidekiq::VERSION} with redis options #{scrubbed_options}")
|
108
111
|
else
|
data/lib/sidekiq/scheduled.rb
CHANGED
@@ -14,18 +14,19 @@ module Sidekiq
|
|
14
14
|
# Just check Redis for the set of jobs with a timestamp before now.
|
15
15
|
Sidekiq.redis do |conn|
|
16
16
|
sorted_sets.each do |sorted_set|
|
17
|
-
# Get
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
|
25
|
-
|
26
|
-
|
27
|
-
|
28
|
-
|
17
|
+
# Get next items in the queue with scores (time to execute) <= now.
|
18
|
+
until (jobs = conn.zrangebyscore(sorted_set, "-inf", now, limit: [0, 100])).empty?
|
19
|
+
# We need to go through the list one at a time to reduce the risk of something
|
20
|
+
# going wrong between the time jobs are popped from the scheduled queue and when
|
21
|
+
# they are pushed onto a work queue and losing the jobs.
|
22
|
+
jobs.each do |job|
|
23
|
+
# Pop item off the queue and add it to the work queue. If the job can't be popped from
|
24
|
+
# the queue, it's because another process already popped it so we can move on to the
|
25
|
+
# next one.
|
26
|
+
if conn.zrem(sorted_set, job)
|
27
|
+
Sidekiq::Client.push(Sidekiq.load_json(job))
|
28
|
+
Sidekiq.logger.debug { "enqueued #{sorted_set}: #{job}" }
|
29
|
+
end
|
29
30
|
end
|
30
31
|
end
|
31
32
|
end
|