sidekiq 5.2.1 → 6.0.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of sidekiq might be problematic. Click here for more details.
- checksums.yaml +5 -5
- data/.circleci/config.yml +60 -0
- data/.gitignore +1 -1
- data/.standard.yml +20 -0
- data/6.0-Upgrade.md +72 -0
- data/COMM-LICENSE +11 -9
- data/Changes.md +209 -0
- data/Ent-2.0-Upgrade.md +37 -0
- data/Ent-Changes.md +36 -1
- data/Gemfile +19 -9
- data/Gemfile.lock +208 -0
- data/Pro-5.0-Upgrade.md +25 -0
- data/Pro-Changes.md +44 -1
- data/README.md +19 -31
- data/Rakefile +6 -4
- data/bin/sidekiq +19 -0
- data/bin/sidekiqload +33 -25
- data/bin/sidekiqmon +8 -0
- data/lib/generators/sidekiq/templates/worker_test.rb.erb +1 -1
- data/lib/generators/sidekiq/worker_generator.rb +21 -13
- data/lib/sidekiq/api.rb +240 -214
- data/lib/sidekiq/cli.rb +167 -219
- data/lib/sidekiq/client.rb +61 -46
- data/lib/sidekiq/delay.rb +5 -6
- data/lib/sidekiq/exception_handler.rb +10 -12
- data/lib/sidekiq/extensions/action_mailer.rb +10 -20
- data/lib/sidekiq/extensions/active_record.rb +9 -7
- data/lib/sidekiq/extensions/class_methods.rb +9 -7
- data/lib/sidekiq/extensions/generic_proxy.rb +4 -4
- data/lib/sidekiq/fetch.rb +11 -12
- data/lib/sidekiq/job_logger.rb +47 -9
- data/lib/sidekiq/job_retry.rb +79 -58
- data/lib/sidekiq/launcher.rb +86 -54
- data/lib/sidekiq/logger.rb +165 -0
- data/lib/sidekiq/manager.rb +10 -12
- data/lib/sidekiq/middleware/chain.rb +14 -4
- data/lib/sidekiq/middleware/i18n.rb +5 -7
- data/lib/sidekiq/monitor.rb +133 -0
- data/lib/sidekiq/paginator.rb +18 -14
- data/lib/sidekiq/processor.rb +113 -79
- data/lib/sidekiq/rails.rb +24 -29
- data/lib/sidekiq/redis_connection.rb +42 -24
- data/lib/sidekiq/scheduled.rb +28 -29
- data/lib/sidekiq/sd_notify.rb +149 -0
- data/lib/sidekiq/systemd.rb +24 -0
- data/lib/sidekiq/testing/inline.rb +2 -1
- data/lib/sidekiq/testing.rb +34 -23
- data/lib/sidekiq/util.rb +17 -16
- data/lib/sidekiq/version.rb +2 -1
- data/lib/sidekiq/web/action.rb +14 -10
- data/lib/sidekiq/web/application.rb +79 -69
- data/lib/sidekiq/web/helpers.rb +89 -71
- data/lib/sidekiq/web/router.rb +17 -16
- data/lib/sidekiq/web.rb +41 -49
- data/lib/sidekiq/worker.rb +134 -91
- data/lib/sidekiq.rb +69 -44
- data/sidekiq.gemspec +16 -18
- data/web/assets/javascripts/application.js +22 -19
- data/web/assets/javascripts/dashboard.js +16 -25
- data/web/assets/stylesheets/application-dark.css +122 -0
- data/web/assets/stylesheets/application.css +44 -2
- data/web/assets/stylesheets/bootstrap.css +1 -1
- data/web/locales/ar.yml +1 -0
- data/web/locales/de.yml +14 -2
- data/web/locales/en.yml +3 -0
- data/web/locales/fr.yml +2 -2
- data/web/locales/ja.yml +4 -1
- data/web/locales/lt.yml +83 -0
- data/web/locales/vi.yml +83 -0
- data/web/views/_job_info.erb +2 -1
- data/web/views/_nav.erb +3 -17
- data/web/views/busy.erb +4 -1
- data/web/views/dead.erb +2 -2
- data/web/views/layout.erb +1 -0
- data/web/views/morgue.erb +4 -1
- data/web/views/queue.erb +11 -1
- data/web/views/queues.erb +9 -1
- data/web/views/retries.erb +8 -1
- data/web/views/retry.erb +2 -2
- data/web/views/scheduled.erb +4 -1
- metadata +37 -27
- data/.travis.yml +0 -14
- data/bin/sidekiqctl +0 -99
- data/lib/sidekiq/core_ext.rb +0 -1
- data/lib/sidekiq/logging.rb +0 -122
- data/lib/sidekiq/middleware/server/active_record.rb +0 -23
data/lib/sidekiq/job_retry.rb
CHANGED
@@ -1,6 +1,10 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
|
-
|
3
|
-
require
|
2
|
+
|
3
|
+
require "sidekiq/scheduled"
|
4
|
+
require "sidekiq/api"
|
5
|
+
|
6
|
+
require "zlib"
|
7
|
+
require "base64"
|
4
8
|
|
5
9
|
module Sidekiq
|
6
10
|
##
|
@@ -56,7 +60,8 @@ module Sidekiq
|
|
56
60
|
# end
|
57
61
|
#
|
58
62
|
class JobRetry
|
59
|
-
class
|
63
|
+
class Handled < ::RuntimeError; end
|
64
|
+
class Skip < Handled; end
|
60
65
|
|
61
66
|
include Sidekiq::Util
|
62
67
|
|
@@ -69,9 +74,9 @@ module Sidekiq
|
|
69
74
|
# The global retry handler requires only the barest of data.
|
70
75
|
# We want to be able to retry as much as possible so we don't
|
71
76
|
# require the worker to be instantiated.
|
72
|
-
def global(
|
77
|
+
def global(jobstr, queue)
|
73
78
|
yield
|
74
|
-
rescue
|
79
|
+
rescue Handled => ex
|
75
80
|
raise ex
|
76
81
|
rescue Sidekiq::Shutdown => ey
|
77
82
|
# ignore, will be pushed back onto queue during hard_shutdown
|
@@ -80,11 +85,19 @@ module Sidekiq
|
|
80
85
|
# ignore, will be pushed back onto queue during hard_shutdown
|
81
86
|
raise Sidekiq::Shutdown if exception_caused_by_shutdown?(e)
|
82
87
|
|
83
|
-
|
84
|
-
|
85
|
-
|
86
|
-
|
88
|
+
msg = Sidekiq.load_json(jobstr)
|
89
|
+
if msg["retry"]
|
90
|
+
attempt_retry(nil, msg, queue, e)
|
91
|
+
else
|
92
|
+
Sidekiq.death_handlers.each do |handler|
|
93
|
+
handler.call(msg, e)
|
94
|
+
rescue => handler_ex
|
95
|
+
handle_exception(handler_ex, {context: "Error calling death handler", job: msg})
|
96
|
+
end
|
97
|
+
end
|
87
98
|
|
99
|
+
raise Handled
|
100
|
+
end
|
88
101
|
|
89
102
|
# The local retry support means that any errors that occur within
|
90
103
|
# this block can be associated with the given worker instance.
|
@@ -94,9 +107,9 @@ module Sidekiq
|
|
94
107
|
# exception so the global block does not reprocess the error. The
|
95
108
|
# Skip exception is unwrapped within Sidekiq::Processor#process before
|
96
109
|
# calling the handle_exception handlers.
|
97
|
-
def local(worker,
|
110
|
+
def local(worker, jobstr, queue)
|
98
111
|
yield
|
99
|
-
rescue
|
112
|
+
rescue Handled => ex
|
100
113
|
raise ex
|
101
114
|
rescue Sidekiq::Shutdown => ey
|
102
115
|
# ignore, will be pushed back onto queue during hard_shutdown
|
@@ -105,11 +118,12 @@ module Sidekiq
|
|
105
118
|
# ignore, will be pushed back onto queue during hard_shutdown
|
106
119
|
raise Sidekiq::Shutdown if exception_caused_by_shutdown?(e)
|
107
120
|
|
108
|
-
|
109
|
-
|
121
|
+
msg = Sidekiq.load_json(jobstr)
|
122
|
+
if msg["retry"].nil?
|
123
|
+
msg["retry"] = worker.class.get_sidekiq_options["retry"]
|
110
124
|
end
|
111
125
|
|
112
|
-
raise e unless msg[
|
126
|
+
raise e unless msg["retry"]
|
113
127
|
attempt_retry(worker, msg, queue, e)
|
114
128
|
# We've handled this error associated with this job, don't
|
115
129
|
# need to handle it at the global level
|
@@ -122,47 +136,44 @@ module Sidekiq
|
|
122
136
|
# instantiate the worker instance. All access must be guarded and
|
123
137
|
# best effort.
|
124
138
|
def attempt_retry(worker, msg, queue, exception)
|
125
|
-
max_retry_attempts = retry_attempts_from(msg[
|
139
|
+
max_retry_attempts = retry_attempts_from(msg["retry"], @max_retries)
|
126
140
|
|
127
|
-
msg[
|
128
|
-
msg['retry_queue']
|
129
|
-
else
|
130
|
-
queue
|
131
|
-
end
|
141
|
+
msg["queue"] = (msg["retry_queue"] || queue)
|
132
142
|
|
133
|
-
|
134
|
-
# that won't convert to JSON.
|
135
|
-
m = exception.message.to_s[0, 10_000]
|
143
|
+
m = exception_message(exception)
|
136
144
|
if m.respond_to?(:scrub!)
|
137
145
|
m.force_encoding("utf-8")
|
138
146
|
m.scrub!
|
139
147
|
end
|
140
148
|
|
141
|
-
msg[
|
142
|
-
msg[
|
143
|
-
count = if msg[
|
144
|
-
msg[
|
145
|
-
msg[
|
149
|
+
msg["error_message"] = m
|
150
|
+
msg["error_class"] = exception.class.name
|
151
|
+
count = if msg["retry_count"]
|
152
|
+
msg["retried_at"] = Time.now.to_f
|
153
|
+
msg["retry_count"] += 1
|
146
154
|
else
|
147
|
-
msg[
|
148
|
-
msg[
|
155
|
+
msg["failed_at"] = Time.now.to_f
|
156
|
+
msg["retry_count"] = 0
|
149
157
|
end
|
150
158
|
|
151
|
-
if msg[
|
152
|
-
msg[
|
153
|
-
|
154
|
-
|
155
|
-
|
156
|
-
|
159
|
+
if msg["backtrace"]
|
160
|
+
lines = if msg["backtrace"] == true
|
161
|
+
exception.backtrace
|
162
|
+
else
|
163
|
+
exception.backtrace[0...msg["backtrace"].to_i]
|
164
|
+
end
|
165
|
+
|
166
|
+
msg["error_backtrace"] = compress_backtrace(lines)
|
157
167
|
end
|
158
168
|
|
159
169
|
if count < max_retry_attempts
|
160
170
|
delay = delay_for(worker, count, exception)
|
161
|
-
|
171
|
+
# Logging here can break retries if the logging device raises ENOSPC #3979
|
172
|
+
# logger.debug { "Failure! Retry #{count} in #{delay} seconds" }
|
162
173
|
retry_at = Time.now.to_f + delay
|
163
174
|
payload = Sidekiq.dump_json(msg)
|
164
175
|
Sidekiq.redis do |conn|
|
165
|
-
conn.zadd(
|
176
|
+
conn.zadd("retry", retry_at.to_s, payload)
|
166
177
|
end
|
167
178
|
else
|
168
179
|
# Goodbye dear message, you (re)tried your best I'm sure.
|
@@ -171,27 +182,24 @@ module Sidekiq
|
|
171
182
|
end
|
172
183
|
|
173
184
|
def retries_exhausted(worker, msg, exception)
|
174
|
-
logger.debug { "Retries exhausted for job" }
|
175
185
|
begin
|
176
|
-
block = worker
|
177
|
-
block
|
186
|
+
block = worker&.sidekiq_retries_exhausted_block
|
187
|
+
block&.call(msg, exception)
|
178
188
|
rescue => e
|
179
|
-
handle_exception(e, {
|
189
|
+
handle_exception(e, {context: "Error calling retries_exhausted", job: msg})
|
180
190
|
end
|
181
191
|
|
192
|
+
send_to_morgue(msg) unless msg["dead"] == false
|
193
|
+
|
182
194
|
Sidekiq.death_handlers.each do |handler|
|
183
|
-
|
184
|
-
|
185
|
-
|
186
|
-
handle_exception(e, { context: "Error calling death handler", job: msg })
|
187
|
-
end
|
195
|
+
handler.call(msg, exception)
|
196
|
+
rescue => e
|
197
|
+
handle_exception(e, {context: "Error calling death handler", job: msg})
|
188
198
|
end
|
189
|
-
|
190
|
-
send_to_morgue(msg) unless msg['dead'] == false
|
191
199
|
end
|
192
200
|
|
193
201
|
def send_to_morgue(msg)
|
194
|
-
|
202
|
+
logger.info { "Adding dead #{msg["class"]} job #{msg["jid"]}" }
|
195
203
|
payload = Sidekiq.dump_json(msg)
|
196
204
|
DeadSet.new.kill(payload, notify_failure: false)
|
197
205
|
end
|
@@ -205,7 +213,7 @@ module Sidekiq
|
|
205
213
|
end
|
206
214
|
|
207
215
|
def delay_for(worker, count, exception)
|
208
|
-
if worker
|
216
|
+
if worker&.sidekiq_retry_in_block
|
209
217
|
custom_retry_in = retry_in(worker, count, exception).to_i
|
210
218
|
return custom_retry_in if custom_retry_in > 0
|
211
219
|
end
|
@@ -214,16 +222,14 @@ module Sidekiq
|
|
214
222
|
|
215
223
|
# delayed_job uses the same basic formula
|
216
224
|
def seconds_to_delay(count)
|
217
|
-
(count
|
225
|
+
(count**4) + 15 + (rand(30) * (count + 1))
|
218
226
|
end
|
219
227
|
|
220
228
|
def retry_in(worker, count, exception)
|
221
|
-
|
222
|
-
|
223
|
-
|
224
|
-
|
225
|
-
nil
|
226
|
-
end
|
229
|
+
worker.sidekiq_retry_in_block.call(count, exception)
|
230
|
+
rescue Exception => e
|
231
|
+
handle_exception(e, {context: "Failure scheduling retry using the defined `sidekiq_retry_in` in #{worker.class.name}, falling back to default"})
|
232
|
+
nil
|
227
233
|
end
|
228
234
|
|
229
235
|
def exception_caused_by_shutdown?(e, checked_causes = [])
|
@@ -237,5 +243,20 @@ module Sidekiq
|
|
237
243
|
exception_caused_by_shutdown?(e.cause, checked_causes)
|
238
244
|
end
|
239
245
|
|
246
|
+
# Extract message from exception.
|
247
|
+
# Set a default if the message raises an error
|
248
|
+
def exception_message(exception)
|
249
|
+
# App code can stuff all sorts of crazy binary data into the error message
|
250
|
+
# that won't convert to JSON.
|
251
|
+
exception.message.to_s[0, 10_000]
|
252
|
+
rescue
|
253
|
+
+"!!! ERROR MESSAGE THREW AN ERROR !!!"
|
254
|
+
end
|
255
|
+
|
256
|
+
def compress_backtrace(backtrace)
|
257
|
+
serialized = Sidekiq.dump_json(backtrace)
|
258
|
+
compressed = Zlib::Deflate.deflate(serialized)
|
259
|
+
Base64.encode64(compressed)
|
260
|
+
end
|
240
261
|
end
|
241
262
|
end
|
data/lib/sidekiq/launcher.rb
CHANGED
@@ -1,19 +1,25 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
|
-
|
3
|
-
require
|
4
|
-
require
|
2
|
+
|
3
|
+
require "sidekiq/manager"
|
4
|
+
require "sidekiq/fetch"
|
5
|
+
require "sidekiq/scheduled"
|
5
6
|
|
6
7
|
module Sidekiq
|
7
|
-
# The Launcher
|
8
|
-
# start, monitor and stop the core Actors in Sidekiq.
|
9
|
-
# If any of these actors die, the Sidekiq process exits
|
10
|
-
# immediately.
|
8
|
+
# The Launcher starts the Manager and Poller threads and provides the process heartbeat.
|
11
9
|
class Launcher
|
12
10
|
include Util
|
13
11
|
|
14
|
-
|
12
|
+
STATS_TTL = 5 * 365 * 24 * 60 * 60 # 5 years
|
15
13
|
|
16
|
-
|
14
|
+
PROCTITLES = [
|
15
|
+
proc { "sidekiq" },
|
16
|
+
proc { Sidekiq::VERSION },
|
17
|
+
proc { |me, data| data["tag"] },
|
18
|
+
proc { |me, data| "[#{Processor::WORKER_STATE.size} of #{data["concurrency"]} busy]" },
|
19
|
+
proc { |me, data| "stopping" if me.stopping? }
|
20
|
+
]
|
21
|
+
|
22
|
+
attr_accessor :manager, :poller, :fetcher
|
17
23
|
|
18
24
|
def initialize(options)
|
19
25
|
@manager = Sidekiq::Manager.new(options)
|
@@ -40,7 +46,7 @@ module Sidekiq
|
|
40
46
|
# return until all work is complete and cleaned up.
|
41
47
|
# It can take up to the timeout to complete.
|
42
48
|
def stop
|
43
|
-
deadline =
|
49
|
+
deadline = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC) + @options[:timeout]
|
44
50
|
|
45
51
|
@done = true
|
46
52
|
@manager.quiet
|
@@ -62,17 +68,64 @@ module Sidekiq
|
|
62
68
|
|
63
69
|
private unless $TESTING
|
64
70
|
|
71
|
+
def start_heartbeat
|
72
|
+
loop do
|
73
|
+
heartbeat
|
74
|
+
sleep 5
|
75
|
+
end
|
76
|
+
Sidekiq.logger.info("Heartbeat stopping...")
|
77
|
+
end
|
78
|
+
|
79
|
+
def clear_heartbeat
|
80
|
+
# Remove record from Redis since we are shutting down.
|
81
|
+
# Note we don't stop the heartbeat thread; if the process
|
82
|
+
# doesn't actually exit, it'll reappear in the Web UI.
|
83
|
+
Sidekiq.redis do |conn|
|
84
|
+
conn.pipelined do
|
85
|
+
conn.srem("processes", identity)
|
86
|
+
conn.unlink("#{identity}:workers")
|
87
|
+
end
|
88
|
+
end
|
89
|
+
rescue
|
90
|
+
# best effort, ignore network errors
|
91
|
+
end
|
92
|
+
|
65
93
|
def heartbeat
|
66
|
-
|
67
|
-
results.compact!
|
68
|
-
$0 = results.join(' ')
|
94
|
+
$0 = PROCTITLES.map { |proc| proc.call(self, to_data) }.compact.join(" ")
|
69
95
|
|
70
96
|
❤
|
71
97
|
end
|
72
98
|
|
99
|
+
def self.flush_stats
|
100
|
+
fails = Processor::FAILURE.reset
|
101
|
+
procd = Processor::PROCESSED.reset
|
102
|
+
return if fails + procd == 0
|
103
|
+
|
104
|
+
nowdate = Time.now.utc.strftime("%Y-%m-%d")
|
105
|
+
begin
|
106
|
+
Sidekiq.redis do |conn|
|
107
|
+
conn.pipelined do
|
108
|
+
conn.incrby("stat:processed", procd)
|
109
|
+
conn.incrby("stat:processed:#{nowdate}", procd)
|
110
|
+
conn.expire("stat:processed:#{nowdate}", STATS_TTL)
|
111
|
+
|
112
|
+
conn.incrby("stat:failed", fails)
|
113
|
+
conn.incrby("stat:failed:#{nowdate}", fails)
|
114
|
+
conn.expire("stat:failed:#{nowdate}", STATS_TTL)
|
115
|
+
end
|
116
|
+
end
|
117
|
+
rescue => ex
|
118
|
+
# we're exiting the process, things might be shut down so don't
|
119
|
+
# try to handle the exception
|
120
|
+
Sidekiq.logger.warn("Unable to flush stats: #{ex}")
|
121
|
+
end
|
122
|
+
end
|
123
|
+
at_exit(&method(:flush_stats))
|
124
|
+
|
73
125
|
def ❤
|
74
126
|
key = identity
|
75
127
|
fails = procd = 0
|
128
|
+
|
76
129
|
begin
|
77
130
|
fails = Processor::FAILURE.reset
|
78
131
|
procd = Processor::PROCESSED.reset
|
@@ -80,6 +133,7 @@ module Sidekiq
|
|
80
133
|
|
81
134
|
workers_key = "#{key}:workers"
|
82
135
|
nowdate = Time.now.utc.strftime("%Y-%m-%d")
|
136
|
+
|
83
137
|
Sidekiq.redis do |conn|
|
84
138
|
conn.multi do
|
85
139
|
conn.incrby("stat:processed", procd)
|
@@ -90,59 +144,52 @@ module Sidekiq
|
|
90
144
|
conn.incrby("stat:failed:#{nowdate}", fails)
|
91
145
|
conn.expire("stat:failed:#{nowdate}", STATS_TTL)
|
92
146
|
|
93
|
-
conn.
|
147
|
+
conn.unlink(workers_key)
|
94
148
|
curstate.each_pair do |tid, hash|
|
95
149
|
conn.hset(workers_key, tid, Sidekiq.dump_json(hash))
|
96
150
|
end
|
97
151
|
conn.expire(workers_key, 60)
|
98
152
|
end
|
99
153
|
end
|
154
|
+
|
100
155
|
fails = procd = 0
|
101
156
|
|
102
|
-
_, exists, _, _, msg = Sidekiq.redis
|
103
|
-
conn.multi
|
104
|
-
conn.sadd(
|
157
|
+
_, exists, _, _, msg = Sidekiq.redis { |conn|
|
158
|
+
conn.multi {
|
159
|
+
conn.sadd("processes", key)
|
105
160
|
conn.exists(key)
|
106
|
-
conn.hmset(key,
|
161
|
+
conn.hmset(key, "info", to_json, "busy", curstate.size, "beat", Time.now.to_f, "quiet", @done)
|
107
162
|
conn.expire(key, 60)
|
108
163
|
conn.rpop("#{key}-signals")
|
109
|
-
|
110
|
-
|
164
|
+
}
|
165
|
+
}
|
111
166
|
|
112
167
|
# first heartbeat or recovering from an outage and need to reestablish our heartbeat
|
113
|
-
fire_event(:heartbeat)
|
168
|
+
fire_event(:heartbeat) unless exists
|
114
169
|
|
115
170
|
return unless msg
|
116
171
|
|
117
|
-
::Process.kill(msg,
|
172
|
+
::Process.kill(msg, ::Process.pid)
|
118
173
|
rescue => e
|
119
174
|
# ignore all redis/network issues
|
120
|
-
logger.error("heartbeat: #{e
|
175
|
+
logger.error("heartbeat: #{e}")
|
121
176
|
# don't lose the counts if there was a network issue
|
122
177
|
Processor::PROCESSED.incr(procd)
|
123
178
|
Processor::FAILURE.incr(fails)
|
124
179
|
end
|
125
180
|
end
|
126
181
|
|
127
|
-
def start_heartbeat
|
128
|
-
while true
|
129
|
-
heartbeat
|
130
|
-
sleep 5
|
131
|
-
end
|
132
|
-
Sidekiq.logger.info("Heartbeat stopping...")
|
133
|
-
end
|
134
|
-
|
135
182
|
def to_data
|
136
183
|
@data ||= begin
|
137
184
|
{
|
138
|
-
|
139
|
-
|
140
|
-
|
141
|
-
|
142
|
-
|
143
|
-
|
144
|
-
|
145
|
-
|
185
|
+
"hostname" => hostname,
|
186
|
+
"started_at" => Time.now.to_f,
|
187
|
+
"pid" => ::Process.pid,
|
188
|
+
"tag" => @options[:tag] || "",
|
189
|
+
"concurrency" => @options[:concurrency],
|
190
|
+
"queues" => @options[:queues].uniq,
|
191
|
+
"labels" => @options[:labels],
|
192
|
+
"identity" => identity
|
146
193
|
}
|
147
194
|
end
|
148
195
|
end
|
@@ -154,20 +201,5 @@ module Sidekiq
|
|
154
201
|
Sidekiq.dump_json(to_data)
|
155
202
|
end
|
156
203
|
end
|
157
|
-
|
158
|
-
def clear_heartbeat
|
159
|
-
# Remove record from Redis since we are shutting down.
|
160
|
-
# Note we don't stop the heartbeat thread; if the process
|
161
|
-
# doesn't actually exit, it'll reappear in the Web UI.
|
162
|
-
Sidekiq.redis do |conn|
|
163
|
-
conn.pipelined do
|
164
|
-
conn.srem('processes', identity)
|
165
|
-
conn.del("#{identity}:workers")
|
166
|
-
end
|
167
|
-
end
|
168
|
-
rescue
|
169
|
-
# best effort, ignore network errors
|
170
|
-
end
|
171
|
-
|
172
204
|
end
|
173
205
|
end
|
@@ -0,0 +1,165 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require "logger"
|
4
|
+
require "time"
|
5
|
+
|
6
|
+
module Sidekiq
|
7
|
+
module Context
|
8
|
+
def self.with(hash)
|
9
|
+
current.merge!(hash)
|
10
|
+
yield
|
11
|
+
ensure
|
12
|
+
hash.each_key { |key| current.delete(key) }
|
13
|
+
end
|
14
|
+
|
15
|
+
def self.current
|
16
|
+
Thread.current[:sidekiq_context] ||= {}
|
17
|
+
end
|
18
|
+
end
|
19
|
+
|
20
|
+
module LoggingUtils
|
21
|
+
LEVELS = {
|
22
|
+
"debug" => 0,
|
23
|
+
"info" => 1,
|
24
|
+
"warn" => 2,
|
25
|
+
"error" => 3,
|
26
|
+
"fatal" => 4
|
27
|
+
}
|
28
|
+
LEVELS.default_proc = proc do |_, level|
|
29
|
+
Sidekiq.logger.warn("Invalid log level: #{level.inspect}")
|
30
|
+
nil
|
31
|
+
end
|
32
|
+
|
33
|
+
def debug?
|
34
|
+
level <= 0
|
35
|
+
end
|
36
|
+
|
37
|
+
def info?
|
38
|
+
level <= 1
|
39
|
+
end
|
40
|
+
|
41
|
+
def warn?
|
42
|
+
level <= 2
|
43
|
+
end
|
44
|
+
|
45
|
+
def error?
|
46
|
+
level <= 3
|
47
|
+
end
|
48
|
+
|
49
|
+
def fatal?
|
50
|
+
level <= 4
|
51
|
+
end
|
52
|
+
|
53
|
+
def local_level
|
54
|
+
Thread.current[:sidekiq_log_level]
|
55
|
+
end
|
56
|
+
|
57
|
+
def local_level=(level)
|
58
|
+
case level
|
59
|
+
when Integer
|
60
|
+
Thread.current[:sidekiq_log_level] = level
|
61
|
+
when Symbol, String
|
62
|
+
Thread.current[:sidekiq_log_level] = LEVELS[level.to_s]
|
63
|
+
when nil
|
64
|
+
Thread.current[:sidekiq_log_level] = nil
|
65
|
+
else
|
66
|
+
raise ArgumentError, "Invalid log level: #{level.inspect}"
|
67
|
+
end
|
68
|
+
end
|
69
|
+
|
70
|
+
def level
|
71
|
+
local_level || super
|
72
|
+
end
|
73
|
+
|
74
|
+
# Change the thread-local level for the duration of the given block.
|
75
|
+
def log_at(level)
|
76
|
+
old_local_level = local_level
|
77
|
+
self.local_level = level
|
78
|
+
yield
|
79
|
+
ensure
|
80
|
+
self.local_level = old_local_level
|
81
|
+
end
|
82
|
+
|
83
|
+
# Redefined to check severity against #level, and thus the thread-local level, rather than +@level+.
|
84
|
+
# FIXME: Remove when the minimum Ruby version supports overriding Logger#level.
|
85
|
+
def add(severity, message = nil, progname = nil, &block)
|
86
|
+
severity ||= ::Logger::UNKNOWN
|
87
|
+
progname ||= @progname
|
88
|
+
|
89
|
+
return true if @logdev.nil? || severity < level
|
90
|
+
|
91
|
+
if message.nil?
|
92
|
+
if block_given?
|
93
|
+
message = yield
|
94
|
+
else
|
95
|
+
message = progname
|
96
|
+
progname = @progname
|
97
|
+
end
|
98
|
+
end
|
99
|
+
|
100
|
+
@logdev.write format_message(format_severity(severity), Time.now, progname, message)
|
101
|
+
end
|
102
|
+
end
|
103
|
+
|
104
|
+
class Logger < ::Logger
|
105
|
+
include LoggingUtils
|
106
|
+
|
107
|
+
def initialize(*args, **kwargs)
|
108
|
+
super
|
109
|
+
self.formatter = Sidekiq.log_formatter
|
110
|
+
end
|
111
|
+
|
112
|
+
module Formatters
|
113
|
+
class Base < ::Logger::Formatter
|
114
|
+
def tid
|
115
|
+
Thread.current["sidekiq_tid"] ||= (Thread.current.object_id ^ ::Process.pid).to_s(36)
|
116
|
+
end
|
117
|
+
|
118
|
+
def ctx
|
119
|
+
Sidekiq::Context.current
|
120
|
+
end
|
121
|
+
|
122
|
+
def format_context
|
123
|
+
if ctx.any?
|
124
|
+
" " + ctx.compact.map { |k, v|
|
125
|
+
case v
|
126
|
+
when Array
|
127
|
+
"#{k}=#{v.join(",")}"
|
128
|
+
else
|
129
|
+
"#{k}=#{v}"
|
130
|
+
end
|
131
|
+
}.join(" ")
|
132
|
+
end
|
133
|
+
end
|
134
|
+
end
|
135
|
+
|
136
|
+
class Pretty < Base
|
137
|
+
def call(severity, time, program_name, message)
|
138
|
+
"#{time.utc.iso8601(3)} pid=#{::Process.pid} tid=#{tid}#{format_context} #{severity}: #{message}\n"
|
139
|
+
end
|
140
|
+
end
|
141
|
+
|
142
|
+
class WithoutTimestamp < Pretty
|
143
|
+
def call(severity, time, program_name, message)
|
144
|
+
"pid=#{::Process.pid} tid=#{tid}#{format_context} #{severity}: #{message}\n"
|
145
|
+
end
|
146
|
+
end
|
147
|
+
|
148
|
+
class JSON < Base
|
149
|
+
def call(severity, time, program_name, message)
|
150
|
+
hash = {
|
151
|
+
ts: time.utc.iso8601(3),
|
152
|
+
pid: ::Process.pid,
|
153
|
+
tid: tid,
|
154
|
+
lvl: severity,
|
155
|
+
msg: message
|
156
|
+
}
|
157
|
+
c = ctx
|
158
|
+
hash["ctx"] = c unless c.empty?
|
159
|
+
|
160
|
+
Sidekiq.dump_json(hash) << "\n"
|
161
|
+
end
|
162
|
+
end
|
163
|
+
end
|
164
|
+
end
|
165
|
+
end
|
data/lib/sidekiq/manager.rb
CHANGED
@@ -1,12 +1,11 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
|
-
require 'sidekiq/util'
|
3
|
-
require 'sidekiq/processor'
|
4
|
-
require 'sidekiq/fetch'
|
5
|
-
require 'thread'
|
6
|
-
require 'set'
|
7
2
|
|
8
|
-
|
3
|
+
require "sidekiq/util"
|
4
|
+
require "sidekiq/processor"
|
5
|
+
require "sidekiq/fetch"
|
6
|
+
require "set"
|
9
7
|
|
8
|
+
module Sidekiq
|
10
9
|
##
|
11
10
|
# The Manager is the central coordination point in Sidekiq, controlling
|
12
11
|
# the lifecycle of the Processors.
|
@@ -27,10 +26,10 @@ module Sidekiq
|
|
27
26
|
attr_reader :workers
|
28
27
|
attr_reader :options
|
29
28
|
|
30
|
-
def initialize(options={})
|
29
|
+
def initialize(options = {})
|
31
30
|
logger.debug { options.inspect }
|
32
31
|
@options = options
|
33
|
-
@count = options[:concurrency] ||
|
32
|
+
@count = options[:concurrency] || 10
|
34
33
|
raise ArgumentError, "Concurrency of #{@count} is not supported" if @count < 1
|
35
34
|
|
36
35
|
@done = false
|
@@ -70,11 +69,11 @@ module Sidekiq
|
|
70
69
|
return if @workers.empty?
|
71
70
|
|
72
71
|
logger.info { "Pausing to allow workers to finish..." }
|
73
|
-
remaining = deadline -
|
72
|
+
remaining = deadline - ::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
|
74
73
|
while remaining > PAUSE_TIME
|
75
74
|
return if @workers.empty?
|
76
75
|
sleep PAUSE_TIME
|
77
|
-
remaining = deadline -
|
76
|
+
remaining = deadline - ::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
|
78
77
|
end
|
79
78
|
return if @workers.empty?
|
80
79
|
|
@@ -113,7 +112,7 @@ module Sidekiq
|
|
113
112
|
end
|
114
113
|
|
115
114
|
if cleanup.size > 0
|
116
|
-
jobs = cleanup.map {|p| p.job }.compact
|
115
|
+
jobs = cleanup.map { |p| p.job }.compact
|
117
116
|
|
118
117
|
logger.warn { "Terminating #{cleanup.size} busy worker threads" }
|
119
118
|
logger.warn { "Work still in progress #{jobs.inspect}" }
|
@@ -132,6 +131,5 @@ module Sidekiq
|
|
132
131
|
processor.kill
|
133
132
|
end
|
134
133
|
end
|
135
|
-
|
136
134
|
end
|
137
135
|
end
|