sidekiq 8.0.2 → 8.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/Changes.md +108 -0
- data/README.md +15 -0
- data/bin/lint-herb +13 -0
- data/lib/active_job/queue_adapters/sidekiq_adapter.rb +104 -58
- data/lib/generators/sidekiq/templates/job.rb.erb +1 -1
- data/lib/sidekiq/api.rb +30 -6
- data/lib/sidekiq/capsule.rb +4 -0
- data/lib/sidekiq/cli.rb +16 -4
- data/lib/sidekiq/client.rb +15 -1
- data/lib/sidekiq/component.rb +2 -1
- data/lib/sidekiq/config.rb +11 -6
- data/lib/sidekiq/fetch.rb +1 -0
- data/lib/sidekiq/job/iterable.rb +33 -14
- data/lib/sidekiq/job.rb +4 -2
- data/lib/sidekiq/job_logger.rb +5 -3
- data/lib/sidekiq/job_retry.rb +23 -8
- data/lib/sidekiq/launcher.rb +18 -9
- data/lib/sidekiq/loader.rb +57 -0
- data/lib/sidekiq/logger.rb +16 -9
- data/lib/sidekiq/metrics/tracking.rb +3 -0
- data/lib/sidekiq/middleware/current_attributes.rb +6 -2
- data/lib/sidekiq/middleware/i18n.rb +2 -0
- data/lib/sidekiq/monitor.rb +4 -8
- data/lib/sidekiq/profiler.rb +17 -3
- data/lib/sidekiq/rails.rb +46 -67
- data/lib/sidekiq/redis_connection.rb +2 -2
- data/lib/sidekiq/ring_buffer.rb +1 -0
- data/lib/sidekiq/scheduled.rb +7 -5
- data/lib/sidekiq/testing.rb +1 -1
- data/lib/sidekiq/transaction_aware_client.rb +13 -5
- data/lib/sidekiq/version.rb +1 -1
- data/lib/sidekiq/web/action.rb +45 -2
- data/lib/sidekiq/web/application.rb +22 -4
- data/lib/sidekiq/web/config.rb +3 -3
- data/lib/sidekiq/web/helpers.rb +26 -29
- data/lib/sidekiq/web.rb +23 -3
- data/lib/sidekiq.rb +5 -0
- data/sidekiq.gemspec +5 -5
- data/web/assets/images/logo.png +0 -0
- data/web/assets/images/status.png +0 -0
- data/web/assets/javascripts/application.js +36 -13
- data/web/assets/javascripts/dashboard.js +1 -1
- data/web/assets/stylesheets/style.css +30 -6
- data/web/locales/ar.yml +1 -0
- data/web/locales/cs.yml +1 -0
- data/web/locales/da.yml +1 -0
- data/web/locales/de.yml +1 -0
- data/web/locales/el.yml +1 -0
- data/web/locales/en.yml +1 -0
- data/web/locales/es.yml +1 -0
- data/web/locales/fa.yml +1 -0
- data/web/locales/fr.yml +2 -1
- data/web/locales/gd.yml +1 -0
- data/web/locales/he.yml +1 -0
- data/web/locales/hi.yml +1 -0
- data/web/locales/it.yml +8 -0
- data/web/locales/ja.yml +1 -0
- data/web/locales/ko.yml +1 -0
- data/web/locales/lt.yml +1 -0
- data/web/locales/nb.yml +1 -0
- data/web/locales/nl.yml +1 -0
- data/web/locales/pl.yml +1 -0
- data/web/locales/pt-BR.yml +1 -0
- data/web/locales/pt.yml +1 -0
- data/web/locales/ru.yml +1 -0
- data/web/locales/sv.yml +1 -0
- data/web/locales/ta.yml +1 -0
- data/web/locales/tr.yml +1 -0
- data/web/locales/uk.yml +6 -5
- data/web/locales/ur.yml +1 -0
- data/web/locales/vi.yml +1 -0
- data/web/locales/zh-CN.yml +1 -0
- data/web/locales/zh-TW.yml +1 -0
- data/web/views/{_footer.erb → _footer.html.erb} +1 -1
- data/web/views/{_metrics_period_select.erb → _metrics_period_select.html.erb} +1 -1
- data/web/views/{_paging.erb → _paging.html.erb} +0 -1
- data/web/views/_poll_link.html.erb +4 -0
- data/web/views/{busy.erb → busy.html.erb} +4 -8
- data/web/views/{dashboard.erb → dashboard.html.erb} +3 -3
- data/web/views/{dead.erb → dead.html.erb} +3 -3
- data/web/views/filtering.html.erb +6 -0
- data/web/views/{layout.erb → layout.html.erb} +8 -7
- data/web/views/{metrics.erb → metrics.html.erb} +9 -8
- data/web/views/{morgue.erb → morgue.html.erb} +8 -4
- data/web/views/{queue.erb → queue.html.erb} +2 -2
- data/web/views/{queues.erb → queues.html.erb} +4 -4
- data/web/views/{retries.erb → retries.html.erb} +9 -5
- data/web/views/{retry.erb → retry.html.erb} +2 -2
- data/web/views/{scheduled.erb → scheduled.html.erb} +9 -5
- data/web/views/{scheduled_job_info.erb → scheduled_job_info.html.erb} +2 -2
- metadata +37 -36
- data/lib/sidekiq/web/csrf_protection.rb +0 -183
- data/web/views/_poll_link.erb +0 -4
- data/web/views/filtering.erb +0 -6
- /data/web/views/{_job_info.erb → _job_info.html.erb} +0 -0
- /data/web/views/{_nav.erb → _nav.html.erb} +0 -0
- /data/web/views/{_summary.erb → _summary.html.erb} +0 -0
- /data/web/views/{metrics_for_job.erb → metrics_for_job.html.erb} +0 -0
- /data/web/views/{profiles.erb → profiles.html.erb} +0 -0
data/lib/sidekiq/job/iterable.rb
CHANGED
|
@@ -32,8 +32,14 @@ module Sidekiq
|
|
|
32
32
|
@_runtime = 0
|
|
33
33
|
@_args = nil
|
|
34
34
|
@_cancelled = nil
|
|
35
|
+
@current_object = nil
|
|
35
36
|
end
|
|
36
37
|
|
|
38
|
+
# Access to the current object while iterating.
|
|
39
|
+
# This value is not reset so the latest element is
|
|
40
|
+
# explicitly available to cleanup/complete callbacks.
|
|
41
|
+
attr_reader :current_object
|
|
42
|
+
|
|
37
43
|
def arguments
|
|
38
44
|
@_args
|
|
39
45
|
end
|
|
@@ -49,7 +55,7 @@ module Sidekiq
|
|
|
49
55
|
def cancel!
|
|
50
56
|
return @_cancelled if cancelled?
|
|
51
57
|
|
|
52
|
-
key =
|
|
58
|
+
key = iteration_key
|
|
53
59
|
_, result, _ = Sidekiq.redis do |c|
|
|
54
60
|
c.pipelined do |p|
|
|
55
61
|
p.hsetnx(key, "cancelled", Time.now.to_i)
|
|
@@ -137,7 +143,7 @@ module Sidekiq
|
|
|
137
143
|
fetch_previous_iteration_state
|
|
138
144
|
|
|
139
145
|
@_executions += 1
|
|
140
|
-
@_start_time =
|
|
146
|
+
@_start_time = mono_now
|
|
141
147
|
|
|
142
148
|
enumerator = build_enumerator(*args, cursor: @_cursor)
|
|
143
149
|
unless enumerator
|
|
@@ -171,7 +177,7 @@ module Sidekiq
|
|
|
171
177
|
private
|
|
172
178
|
|
|
173
179
|
def is_cancelled?
|
|
174
|
-
@_cancelled = Sidekiq.redis { |c| c.hget(
|
|
180
|
+
@_cancelled = Sidekiq.redis { |c| c.hget(iteration_key, "cancelled") }
|
|
175
181
|
end
|
|
176
182
|
|
|
177
183
|
def fetch_previous_iteration_state
|
|
@@ -198,16 +204,17 @@ module Sidekiq
|
|
|
198
204
|
|
|
199
205
|
time_limit = Sidekiq.default_configuration[:timeout]
|
|
200
206
|
found_record = false
|
|
201
|
-
state_flushed_at =
|
|
207
|
+
state_flushed_at = mono_now
|
|
202
208
|
|
|
203
209
|
enumerator.each do |object, cursor|
|
|
204
210
|
found_record = true
|
|
205
211
|
@_cursor = cursor
|
|
212
|
+
@current_object = object
|
|
206
213
|
|
|
207
|
-
|
|
208
|
-
if
|
|
214
|
+
interrupt_job = interrupted? || should_interrupt?
|
|
215
|
+
if mono_now - state_flushed_at >= STATE_FLUSH_INTERVAL || interrupt_job
|
|
209
216
|
_, _, cancelled = flush_state
|
|
210
|
-
state_flushed_at =
|
|
217
|
+
state_flushed_at = mono_now
|
|
211
218
|
if cancelled
|
|
212
219
|
@_cancelled = true
|
|
213
220
|
on_cancel
|
|
@@ -216,11 +223,14 @@ module Sidekiq
|
|
|
216
223
|
end
|
|
217
224
|
end
|
|
218
225
|
|
|
219
|
-
return false if
|
|
226
|
+
return false if interrupt_job
|
|
220
227
|
|
|
221
|
-
verify_iteration_time(time_limit
|
|
228
|
+
verify_iteration_time(time_limit) do
|
|
222
229
|
around_iteration do
|
|
223
230
|
each_iteration(object, *arguments)
|
|
231
|
+
rescue Exception
|
|
232
|
+
flush_state
|
|
233
|
+
raise
|
|
224
234
|
end
|
|
225
235
|
end
|
|
226
236
|
end
|
|
@@ -228,16 +238,16 @@ module Sidekiq
|
|
|
228
238
|
logger.debug("Enumerator found nothing to iterate!") unless found_record
|
|
229
239
|
true
|
|
230
240
|
ensure
|
|
231
|
-
@_runtime += (
|
|
241
|
+
@_runtime += (mono_now - @_start_time)
|
|
232
242
|
end
|
|
233
243
|
|
|
234
|
-
def verify_iteration_time(time_limit
|
|
235
|
-
start =
|
|
244
|
+
def verify_iteration_time(time_limit)
|
|
245
|
+
start = mono_now
|
|
236
246
|
yield
|
|
237
|
-
finish =
|
|
247
|
+
finish = mono_now
|
|
238
248
|
total = finish - start
|
|
239
249
|
if total > time_limit
|
|
240
|
-
logger.warn { "Iteration took longer (%.2f) than Sidekiq's shutdown timeout (%d)
|
|
250
|
+
logger.warn { "Iteration took longer (%.2f) than Sidekiq's shutdown timeout (%d). This can lead to job processing problems during deploys" % [total, time_limit] }
|
|
241
251
|
end
|
|
242
252
|
end
|
|
243
253
|
|
|
@@ -263,6 +273,11 @@ module Sidekiq
|
|
|
263
273
|
end
|
|
264
274
|
end
|
|
265
275
|
|
|
276
|
+
def should_interrupt?
|
|
277
|
+
max_iteration_runtime = Sidekiq.default_configuration[:max_iteration_runtime]
|
|
278
|
+
max_iteration_runtime && (mono_now - @_start_time > max_iteration_runtime)
|
|
279
|
+
end
|
|
280
|
+
|
|
266
281
|
def flush_state
|
|
267
282
|
key = iteration_key
|
|
268
283
|
state = {
|
|
@@ -298,6 +313,10 @@ module Sidekiq
|
|
|
298
313
|
raise "Unexpected thrown value: #{completed.inspect}"
|
|
299
314
|
end
|
|
300
315
|
end
|
|
316
|
+
|
|
317
|
+
def mono_now
|
|
318
|
+
::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
|
|
319
|
+
end
|
|
301
320
|
end
|
|
302
321
|
end
|
|
303
322
|
end
|
data/lib/sidekiq/job.rb
CHANGED
|
@@ -226,6 +226,8 @@ module Sidekiq
|
|
|
226
226
|
end
|
|
227
227
|
return nil unless result
|
|
228
228
|
|
|
229
|
+
verify_json(item)
|
|
230
|
+
|
|
229
231
|
# round-trip the payload via JSON
|
|
230
232
|
msg = Sidekiq.load_json(Sidekiq.dump_json(item))
|
|
231
233
|
|
|
@@ -248,9 +250,9 @@ module Sidekiq
|
|
|
248
250
|
end
|
|
249
251
|
alias_method :perform_sync, :perform_inline
|
|
250
252
|
|
|
251
|
-
def perform_bulk(args,
|
|
253
|
+
def perform_bulk(args, **options)
|
|
252
254
|
client = @klass.build_client
|
|
253
|
-
client.push_bulk(@opts.merge("class" => @klass, "args" => args,
|
|
255
|
+
client.push_bulk(@opts.merge({"class" => @klass, "args" => args}, options))
|
|
254
256
|
end
|
|
255
257
|
|
|
256
258
|
# +interval+ must be a timestamp, numeric or something that acts
|
data/lib/sidekiq/job_logger.rb
CHANGED
|
@@ -27,10 +27,12 @@ module Sidekiq
|
|
|
27
27
|
# attribute to expose the underlying thing.
|
|
28
28
|
h = {
|
|
29
29
|
jid: job_hash["jid"],
|
|
30
|
-
class: job_hash["
|
|
30
|
+
class: job_hash["wrapped"] || job_hash["class"]
|
|
31
31
|
}
|
|
32
|
-
|
|
33
|
-
|
|
32
|
+
|
|
33
|
+
@config[:logged_job_attributes].each do |attr|
|
|
34
|
+
h[attr.to_sym] = job_hash[attr] if job_hash.has_key?(attr)
|
|
35
|
+
end
|
|
34
36
|
|
|
35
37
|
Thread.current[:sidekiq_context] = h
|
|
36
38
|
level = job_hash["log_level"]
|
data/lib/sidekiq/job_retry.rb
CHANGED
|
@@ -178,15 +178,21 @@ module Sidekiq
|
|
|
178
178
|
msg["error_backtrace"] = compress_backtrace(lines)
|
|
179
179
|
end
|
|
180
180
|
|
|
181
|
-
|
|
182
|
-
|
|
181
|
+
# retry_for and retry are mutually exclusive - if retry_for is set,
|
|
182
|
+
# we exclusively use duration-based retry logic and ignore count-based logic
|
|
183
183
|
rf = msg["retry_for"]
|
|
184
|
-
|
|
184
|
+
if rf
|
|
185
|
+
return retries_exhausted(jobinst, msg, exception) if (time_for(msg["failed_at"]) + rf) < Time.now
|
|
186
|
+
elsif count >= max_retry_attempts
|
|
187
|
+
return retries_exhausted(jobinst, msg, exception)
|
|
188
|
+
end
|
|
185
189
|
|
|
186
190
|
strategy, delay = delay_for(jobinst, count, exception, msg)
|
|
187
191
|
case strategy
|
|
188
192
|
when :discard
|
|
189
|
-
|
|
193
|
+
msg["discarded_at"] = now_ms
|
|
194
|
+
|
|
195
|
+
return run_death_handlers(msg, exception)
|
|
190
196
|
when :kill
|
|
191
197
|
return retries_exhausted(jobinst, msg, exception)
|
|
192
198
|
end
|
|
@@ -255,13 +261,22 @@ module Sidekiq
|
|
|
255
261
|
handle_exception(e, {context: "Error calling retries_exhausted", job: msg})
|
|
256
262
|
end
|
|
257
263
|
|
|
258
|
-
|
|
259
|
-
|
|
264
|
+
discarded = msg["dead"] == false || rv == :discard
|
|
265
|
+
|
|
266
|
+
if discarded
|
|
267
|
+
msg["discarded_at"] = now_ms
|
|
268
|
+
else
|
|
269
|
+
send_to_morgue(msg)
|
|
270
|
+
end
|
|
271
|
+
|
|
272
|
+
run_death_handlers(msg, exception)
|
|
273
|
+
end
|
|
260
274
|
|
|
275
|
+
def run_death_handlers(job, exception)
|
|
261
276
|
@capsule.config.death_handlers.each do |handler|
|
|
262
|
-
handler.call(
|
|
277
|
+
handler.call(job, exception)
|
|
263
278
|
rescue => e
|
|
264
|
-
handle_exception(e, {context: "Error calling death handler", job:
|
|
279
|
+
handle_exception(e, {context: "Error calling death handler", job: job})
|
|
265
280
|
end
|
|
266
281
|
end
|
|
267
282
|
|
data/lib/sidekiq/launcher.rb
CHANGED
|
@@ -142,6 +142,12 @@ module Sidekiq
|
|
|
142
142
|
key = identity
|
|
143
143
|
fails = procd = 0
|
|
144
144
|
|
|
145
|
+
idle_timeout = config[:redis_idle_timeout]
|
|
146
|
+
if idle_timeout
|
|
147
|
+
config.capsules.each_value { |cap| cap.local_redis_pool.reap(idle_seconds: idle_timeout, &:close) }
|
|
148
|
+
config.local_redis_pool.reap(idle_seconds: idle_timeout, &:close)
|
|
149
|
+
end
|
|
150
|
+
|
|
145
151
|
begin
|
|
146
152
|
flush_stats
|
|
147
153
|
|
|
@@ -214,11 +220,11 @@ module Sidekiq
|
|
|
214
220
|
# Log a warning if it's a disaster.
|
|
215
221
|
if RTT_READINGS.all? { |x| x > RTT_WARNING_LEVEL }
|
|
216
222
|
logger.warn <<~EOM
|
|
217
|
-
Your Redis network connection
|
|
223
|
+
Your Redis network connection appears to be performing poorly.
|
|
218
224
|
Last RTT readings were #{RTT_READINGS.buffer.inspect}, ideally these should be < 1000.
|
|
219
|
-
Ensure Redis is running in the same AZ or datacenter as Sidekiq
|
|
220
|
-
|
|
221
|
-
|
|
225
|
+
Ensure Redis is running in the same AZ or datacenter as Sidekiq and that
|
|
226
|
+
your Sidekiq process is not CPU-saturated; reduce your concurrency and/or
|
|
227
|
+
see https://github.com/sidekiq/sidekiq/discussions/5039
|
|
222
228
|
EOM
|
|
223
229
|
RTT_READINGS.reset
|
|
224
230
|
end
|
|
@@ -252,8 +258,15 @@ module Sidekiq
|
|
|
252
258
|
"pid" => ::Process.pid,
|
|
253
259
|
"tag" => @config[:tag] || "",
|
|
254
260
|
"concurrency" => @config.total_concurrency,
|
|
261
|
+
"capsules" => @config.capsules.each_with_object({}) { |(name, cap), memo|
|
|
262
|
+
memo[name] = cap.to_h
|
|
263
|
+
},
|
|
264
|
+
#####
|
|
265
|
+
# TODO deprecated, remove in 9.0
|
|
266
|
+
# This data is now found in the `capsules` element above
|
|
255
267
|
"queues" => @config.capsules.values.flat_map { |cap| cap.queues }.uniq,
|
|
256
|
-
"weights" =>
|
|
268
|
+
"weights" => @config.capsules.values.map(&:weights),
|
|
269
|
+
#####
|
|
257
270
|
"labels" => @config[:labels].to_a,
|
|
258
271
|
"identity" => identity,
|
|
259
272
|
"version" => Sidekiq::VERSION,
|
|
@@ -261,10 +274,6 @@ module Sidekiq
|
|
|
261
274
|
}
|
|
262
275
|
end
|
|
263
276
|
|
|
264
|
-
def to_weights
|
|
265
|
-
@config.capsules.values.map(&:weights)
|
|
266
|
-
end
|
|
267
|
-
|
|
268
277
|
def to_json
|
|
269
278
|
# this data changes infrequently so dump it to a string
|
|
270
279
|
# now so we don't need to dump it every heartbeat.
|
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
module Sidekiq
|
|
2
|
+
require "sidekiq/component"
|
|
3
|
+
|
|
4
|
+
class Loader
|
|
5
|
+
include Sidekiq::Component
|
|
6
|
+
|
|
7
|
+
def initialize(cfg = Sidekiq.default_configuration)
|
|
8
|
+
@config = cfg
|
|
9
|
+
@load_hooks = Hash.new { |h, k| h[k] = [] }
|
|
10
|
+
@loaded = Set.new
|
|
11
|
+
@lock = Mutex.new
|
|
12
|
+
end
|
|
13
|
+
|
|
14
|
+
# Declares a block that will be executed when a Sidekiq component is fully
|
|
15
|
+
# loaded. If the component has already loaded, the block is executed
|
|
16
|
+
# immediately.
|
|
17
|
+
#
|
|
18
|
+
# Sidekiq.loader.on_load(:api) do
|
|
19
|
+
# # extend the sidekiq API
|
|
20
|
+
# end
|
|
21
|
+
#
|
|
22
|
+
def on_load(name, &block)
|
|
23
|
+
# we don't want to hold the lock while calling the block
|
|
24
|
+
to_run = nil
|
|
25
|
+
|
|
26
|
+
@lock.synchronize do
|
|
27
|
+
if @loaded.include?(name)
|
|
28
|
+
to_run = block
|
|
29
|
+
else
|
|
30
|
+
@load_hooks[name] << block
|
|
31
|
+
end
|
|
32
|
+
end
|
|
33
|
+
|
|
34
|
+
to_run&.call
|
|
35
|
+
nil
|
|
36
|
+
end
|
|
37
|
+
|
|
38
|
+
# Executes all blocks registered to +name+ via on_load.
|
|
39
|
+
#
|
|
40
|
+
# Sidekiq.loader.run_load_hooks(:api)
|
|
41
|
+
#
|
|
42
|
+
# In the case of the above example, it will execute all hooks registered for +:api+.
|
|
43
|
+
#
|
|
44
|
+
def run_load_hooks(name)
|
|
45
|
+
hks = @lock.synchronize do
|
|
46
|
+
@loaded << name
|
|
47
|
+
@load_hooks.delete(name)
|
|
48
|
+
end
|
|
49
|
+
|
|
50
|
+
hks&.each do |blk|
|
|
51
|
+
blk.call
|
|
52
|
+
rescue => ex
|
|
53
|
+
handle_exception(ex, hook: name)
|
|
54
|
+
end
|
|
55
|
+
end
|
|
56
|
+
end
|
|
57
|
+
end
|
data/lib/sidekiq/logger.rb
CHANGED
|
@@ -24,14 +24,15 @@ module Sidekiq
|
|
|
24
24
|
|
|
25
25
|
class Logger < ::Logger
|
|
26
26
|
module Formatters
|
|
27
|
-
COLORS = {
|
|
28
|
-
"DEBUG" => "\e[1;32mDEBUG\e[0m", # green
|
|
29
|
-
"INFO" => "\e[1;34mINFO \e[0m", # blue
|
|
30
|
-
"WARN" => "\e[1;33mWARN \e[0m", # yellow
|
|
31
|
-
"ERROR" => "\e[1;31mERROR\e[0m", # red
|
|
32
|
-
"FATAL" => "\e[1;35mFATAL\e[0m" # pink
|
|
33
|
-
}
|
|
34
27
|
class Base < ::Logger::Formatter
|
|
28
|
+
COLORS = {
|
|
29
|
+
"DEBUG" => "\e[1;32mDEBUG\e[0m", # green
|
|
30
|
+
"INFO" => "\e[1;34mINFO \e[0m", # blue
|
|
31
|
+
"WARN" => "\e[1;33mWARN \e[0m", # yellow
|
|
32
|
+
"ERROR" => "\e[1;31mERROR\e[0m", # red
|
|
33
|
+
"FATAL" => "\e[1;35mFATAL\e[0m" # pink
|
|
34
|
+
}
|
|
35
|
+
|
|
35
36
|
def tid
|
|
36
37
|
Thread.current["sidekiq_tid"] ||= (Thread.current.object_id ^ ::Process.pid).to_s(36)
|
|
37
38
|
end
|
|
@@ -50,13 +51,19 @@ module Sidekiq
|
|
|
50
51
|
|
|
51
52
|
class Pretty < Base
|
|
52
53
|
def call(severity, time, program_name, message)
|
|
53
|
-
"#{
|
|
54
|
+
"#{COLORS[severity]} #{time.utc.iso8601(3)} pid=#{::Process.pid} tid=#{tid}#{format_context}: #{message}\n"
|
|
55
|
+
end
|
|
56
|
+
end
|
|
57
|
+
|
|
58
|
+
class Plain < Base
|
|
59
|
+
def call(severity, time, program_name, message)
|
|
60
|
+
"#{severity} #{time.utc.iso8601(3)} pid=#{::Process.pid} tid=#{tid}#{format_context}: #{message}\n"
|
|
54
61
|
end
|
|
55
62
|
end
|
|
56
63
|
|
|
57
64
|
class WithoutTimestamp < Pretty
|
|
58
65
|
def call(severity, time, program_name, message)
|
|
59
|
-
"#{
|
|
66
|
+
"#{COLORS[severity]} pid=#{::Process.pid} tid=#{tid}#{format_context}: #{message}\n"
|
|
60
67
|
end
|
|
61
68
|
end
|
|
62
69
|
|
|
@@ -50,7 +50,7 @@ module Sidekiq
|
|
|
50
50
|
@cattrs = cattrs
|
|
51
51
|
end
|
|
52
52
|
|
|
53
|
-
def call(_, job,
|
|
53
|
+
def call(_, job, *, &block)
|
|
54
54
|
klass_attrs = {}
|
|
55
55
|
|
|
56
56
|
@cattrs.each do |(key, strklass)|
|
|
@@ -71,11 +71,14 @@ module Sidekiq
|
|
|
71
71
|
retried = false
|
|
72
72
|
|
|
73
73
|
begin
|
|
74
|
+
set_succeeded = false
|
|
74
75
|
klass.set(attrs) do
|
|
76
|
+
set_succeeded = true
|
|
75
77
|
wrap(klass_attrs, &block)
|
|
76
78
|
end
|
|
77
79
|
rescue NoMethodError
|
|
78
|
-
|
|
80
|
+
# Don't retry if the no method error didn't come from current attributes
|
|
81
|
+
raise if retried || set_succeeded
|
|
79
82
|
|
|
80
83
|
# It is possible that the `CurrentAttributes` definition
|
|
81
84
|
# was changed before the job started processing.
|
|
@@ -90,6 +93,7 @@ module Sidekiq
|
|
|
90
93
|
def persist(klass_or_array, config = Sidekiq.default_configuration)
|
|
91
94
|
cattrs = build_cattrs_hash(klass_or_array)
|
|
92
95
|
|
|
96
|
+
config.client_middleware.prepend Load, cattrs
|
|
93
97
|
config.client_middleware.add Save, cattrs
|
|
94
98
|
config.server_middleware.prepend Load, cattrs
|
|
95
99
|
end
|
|
@@ -11,6 +11,7 @@ module Sidekiq::Middleware::I18n
|
|
|
11
11
|
# to be sent to Sidekiq.
|
|
12
12
|
class Client
|
|
13
13
|
include Sidekiq::ClientMiddleware
|
|
14
|
+
|
|
14
15
|
def call(_jobclass, job, _queue, _redis)
|
|
15
16
|
job["locale"] ||= I18n.locale
|
|
16
17
|
yield
|
|
@@ -20,6 +21,7 @@ module Sidekiq::Middleware::I18n
|
|
|
20
21
|
# Pull the msg locale out and set the current thread to use it.
|
|
21
22
|
class Server
|
|
22
23
|
include Sidekiq::ServerMiddleware
|
|
24
|
+
|
|
23
25
|
def call(_jobclass, job, _queue, &block)
|
|
24
26
|
I18n.with_locale(job.fetch("locale", I18n.default_locale), &block)
|
|
25
27
|
end
|
data/lib/sidekiq/monitor.rb
CHANGED
|
@@ -49,14 +49,10 @@ class Sidekiq::Monitor
|
|
|
49
49
|
puts "---- Processes (#{process_set.size}) ----"
|
|
50
50
|
process_set.each_with_index do |process, index|
|
|
51
51
|
# Keep compatibility with legacy versions since we don't want to break sidekiqmon during rolling upgrades or downgrades.
|
|
52
|
-
#
|
|
53
|
-
# Before:
|
|
54
|
-
# ["default", "critical"]
|
|
55
|
-
#
|
|
56
|
-
# After:
|
|
57
|
-
# {"default" => 1, "critical" => 10}
|
|
58
52
|
queues =
|
|
59
|
-
if process["
|
|
53
|
+
if process["capsules"] # 8.0.6+
|
|
54
|
+
process["capsules"].values.map { |x| x["weights"].keys.join(", ") }
|
|
55
|
+
elsif process["weights"]
|
|
60
56
|
process["weights"].sort_by { |queue| queue[0] }.map { |capsule| capsule.map { |name, weight| (weight > 0) ? "#{name}: #{weight}" : name }.join(", ") }
|
|
61
57
|
else
|
|
62
58
|
process["queues"].sort
|
|
@@ -105,7 +101,7 @@ class Sidekiq::Monitor
|
|
|
105
101
|
out << line
|
|
106
102
|
line = " " * pad
|
|
107
103
|
end
|
|
108
|
-
line << value + "
|
|
104
|
+
line << value + "; "
|
|
109
105
|
end
|
|
110
106
|
out << line[0..-3]
|
|
111
107
|
out.join("\n")
|
data/lib/sidekiq/profiler.rb
CHANGED
|
@@ -11,8 +11,10 @@ module Sidekiq
|
|
|
11
11
|
}
|
|
12
12
|
|
|
13
13
|
include Sidekiq::Component
|
|
14
|
+
|
|
14
15
|
def initialize(config)
|
|
15
16
|
@config = config
|
|
17
|
+
@vernier_output_dir = ENV.fetch("VERNIER_OUTPUT_DIR") { Dir.tmpdir }
|
|
16
18
|
end
|
|
17
19
|
|
|
18
20
|
def call(job, &block)
|
|
@@ -22,7 +24,6 @@ module Sidekiq
|
|
|
22
24
|
type = job["class"]
|
|
23
25
|
jid = job["jid"]
|
|
24
26
|
started_at = Time.now
|
|
25
|
-
options = DEFAULT_OPTIONS.merge((job["profiler_options"] || {}).transform_keys!(&:to_sym))
|
|
26
27
|
|
|
27
28
|
rundata = {
|
|
28
29
|
started_at: started_at.to_i,
|
|
@@ -30,13 +31,17 @@ module Sidekiq
|
|
|
30
31
|
type: type,
|
|
31
32
|
jid: jid,
|
|
32
33
|
# .gz extension tells Vernier to compress the data
|
|
33
|
-
filename:
|
|
34
|
+
filename: File.join(
|
|
35
|
+
@vernier_output_dir,
|
|
36
|
+
"#{token}-#{type}-#{jid}-#{started_at.strftime("%Y%m%d-%H%M%S")}.json.gz"
|
|
37
|
+
)
|
|
34
38
|
}
|
|
39
|
+
profiler_options = profiler_options(job, rundata)
|
|
35
40
|
|
|
36
41
|
require "vernier"
|
|
37
42
|
begin
|
|
38
43
|
a = Time.now
|
|
39
|
-
rc = Vernier.profile(**
|
|
44
|
+
rc = Vernier.profile(**profiler_options, &block)
|
|
40
45
|
b = Time.now
|
|
41
46
|
|
|
42
47
|
# Failed jobs will raise an exception on previous line and skip this
|
|
@@ -55,5 +60,14 @@ module Sidekiq
|
|
|
55
60
|
FileUtils.rm_f(rundata[:filename])
|
|
56
61
|
end
|
|
57
62
|
end
|
|
63
|
+
|
|
64
|
+
private
|
|
65
|
+
|
|
66
|
+
def profiler_options(job, rundata)
|
|
67
|
+
profiler_options = (job["profiler_options"] || {}).transform_keys(&:to_sym)
|
|
68
|
+
profiler_options[:mode] = profiler_options[:mode].to_sym if profiler_options[:mode]
|
|
69
|
+
|
|
70
|
+
DEFAULT_OPTIONS.merge(profiler_options, {out: rundata[:filename]})
|
|
71
|
+
end
|
|
58
72
|
end
|
|
59
73
|
end
|
data/lib/sidekiq/rails.rb
CHANGED
|
@@ -1,84 +1,63 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
-
require "sidekiq/job"
|
|
4
|
-
require "rails"
|
|
5
|
-
|
|
6
3
|
module Sidekiq
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
class Rails < ::Rails::Engine
|
|
19
|
-
class Reloader
|
|
20
|
-
def initialize(app = ::Rails.application)
|
|
21
|
-
@app = app
|
|
22
|
-
end
|
|
23
|
-
|
|
24
|
-
def call
|
|
25
|
-
params = (::Rails::VERSION::STRING >= "7.1") ? {source: "job.sidekiq"} : {}
|
|
26
|
-
@app.reloader.wrap(**params) do
|
|
27
|
-
yield
|
|
4
|
+
begin
|
|
5
|
+
gem "railties", ">= 7.0"
|
|
6
|
+
require "rails"
|
|
7
|
+
require "sidekiq/job"
|
|
8
|
+
require_relative "../active_job/queue_adapters/sidekiq_adapter"
|
|
9
|
+
|
|
10
|
+
class Rails < ::Rails::Engine
|
|
11
|
+
class Reloader
|
|
12
|
+
def initialize(app = ::Rails.application)
|
|
13
|
+
@app = app
|
|
28
14
|
end
|
|
29
|
-
end
|
|
30
15
|
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
16
|
+
def call
|
|
17
|
+
params = (::Rails::VERSION::STRING >= "7.1") ? {source: "job.sidekiq"} : {}
|
|
18
|
+
@app.reloader.wrap(**params) do
|
|
19
|
+
yield
|
|
20
|
+
end
|
|
21
|
+
end
|
|
34
22
|
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
end
|
|
23
|
+
def inspect
|
|
24
|
+
"#<Sidekiq::Rails::Reloader @app=#{@app.class.name}>"
|
|
25
|
+
end
|
|
39
26
|
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
# manually retried, don't automatically die, etc.
|
|
44
|
-
#
|
|
45
|
-
# class SomeJob < ActiveJob::Base
|
|
46
|
-
# queue_as :default
|
|
47
|
-
# sidekiq_options retry: 3, backtrace: 10
|
|
48
|
-
# def perform
|
|
49
|
-
# end
|
|
50
|
-
# end
|
|
51
|
-
initializer "sidekiq.active_job_integration" do
|
|
52
|
-
ActiveSupport.on_load(:active_job) do
|
|
53
|
-
require_relative "../active_job/queue_adapters/sidekiq_adapter"
|
|
54
|
-
include ::Sidekiq::Job::Options unless respond_to?(:sidekiq_options)
|
|
27
|
+
def to_hash
|
|
28
|
+
{app: @app.class.name}
|
|
29
|
+
end
|
|
55
30
|
end
|
|
56
|
-
end
|
|
57
31
|
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
32
|
+
initializer "sidekiq.backtrace_cleaner" do
|
|
33
|
+
Sidekiq.configure_server do |config|
|
|
34
|
+
config[:backtrace_cleaner] = ->(backtrace) { ::Rails.backtrace_cleaner.clean(backtrace) }
|
|
35
|
+
end
|
|
61
36
|
end
|
|
62
|
-
end
|
|
63
|
-
|
|
64
|
-
# This hook happens after all initializers are run, just before returning
|
|
65
|
-
# from config/environment.rb back to sidekiq/cli.rb.
|
|
66
|
-
#
|
|
67
|
-
# None of this matters on the client-side, only within the Sidekiq process itself.
|
|
68
|
-
config.after_initialize do
|
|
69
|
-
Sidekiq.configure_server do |config|
|
|
70
|
-
config[:reloader] = Sidekiq::Rails::Reloader.new
|
|
71
37
|
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
38
|
+
# This hook happens after all initializers are run, just before returning
|
|
39
|
+
# from config/environment.rb back to sidekiq/cli.rb.
|
|
40
|
+
#
|
|
41
|
+
# None of this matters on the client-side, only within the Sidekiq process itself.
|
|
42
|
+
config.after_initialize do
|
|
43
|
+
Sidekiq.configure_server do |config|
|
|
44
|
+
config[:reloader] = Sidekiq::Rails::Reloader.new
|
|
45
|
+
|
|
46
|
+
# This is the integration code necessary so that if a job uses `Rails.logger.info "Hello"`,
|
|
47
|
+
# it will appear in the Sidekiq console with all of the job context.
|
|
48
|
+
unless ::Rails.logger == config.logger || ::ActiveSupport::Logger.logger_outputs_to?(::Rails.logger, $stdout)
|
|
49
|
+
if ::Rails.logger.respond_to?(:broadcast_to)
|
|
50
|
+
::Rails.logger.broadcast_to(config.logger)
|
|
51
|
+
elsif ::ActiveSupport::Logger.respond_to?(:broadcast)
|
|
52
|
+
::Rails.logger.extend(::ActiveSupport::Logger.broadcast(config.logger))
|
|
53
|
+
else
|
|
54
|
+
::Rails.logger = ::ActiveSupport::BroadcastLogger.new(::Rails.logger, config.logger)
|
|
55
|
+
end
|
|
79
56
|
end
|
|
80
57
|
end
|
|
81
58
|
end
|
|
82
59
|
end
|
|
60
|
+
rescue Gem::LoadError
|
|
61
|
+
# Rails not available or version requirement not met
|
|
83
62
|
end
|
|
84
63
|
end
|
|
@@ -23,7 +23,7 @@ module Sidekiq
|
|
|
23
23
|
|
|
24
24
|
size = symbolized_options.delete(:size) || 5
|
|
25
25
|
pool_timeout = symbolized_options.delete(:pool_timeout) || 1
|
|
26
|
-
|
|
26
|
+
symbolized_options.delete(:pool_name)
|
|
27
27
|
|
|
28
28
|
# Default timeout in redis-client is 1 second, which can be too aggressive
|
|
29
29
|
# if the Sidekiq process is CPU-bound. With 10-15 threads and a thread quantum of 100ms,
|
|
@@ -33,7 +33,7 @@ module Sidekiq
|
|
|
33
33
|
symbolized_options[:timeout] ||= 3
|
|
34
34
|
|
|
35
35
|
redis_config = Sidekiq::RedisClientAdapter.new(symbolized_options)
|
|
36
|
-
ConnectionPool.new(timeout: pool_timeout, size: size
|
|
36
|
+
ConnectionPool.new(timeout: pool_timeout, size: size) do
|
|
37
37
|
redis_config.new_client
|
|
38
38
|
end
|
|
39
39
|
end
|