sidekiq 7.1.0 → 7.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/Changes.md +89 -0
- data/README.md +2 -2
- data/lib/sidekiq/api.rb +7 -7
- data/lib/sidekiq/cli.rb +1 -0
- data/lib/sidekiq/client.rb +8 -3
- data/lib/sidekiq/config.rb +19 -6
- data/lib/sidekiq/deploy.rb +1 -1
- data/lib/sidekiq/job_retry.rb +19 -3
- data/lib/sidekiq/job_util.rb +2 -0
- data/lib/sidekiq/metrics/query.rb +3 -1
- data/lib/sidekiq/metrics/shared.rb +1 -1
- data/lib/sidekiq/middleware/current_attributes.rb +55 -16
- data/lib/sidekiq/paginator.rb +2 -2
- data/lib/sidekiq/processor.rb +27 -26
- data/lib/sidekiq/rails.rb +10 -11
- data/lib/sidekiq/redis_client_adapter.rb +17 -2
- data/lib/sidekiq/redis_connection.rb +1 -0
- data/lib/sidekiq/scheduled.rb +1 -1
- data/lib/sidekiq/testing.rb +25 -6
- data/lib/sidekiq/version.rb +1 -1
- data/lib/sidekiq/web/action.rb +3 -3
- data/lib/sidekiq/web/application.rb +72 -6
- data/lib/sidekiq/web/csrf_protection.rb +1 -1
- data/lib/sidekiq/web/helpers.rb +31 -23
- data/lib/sidekiq/web.rb +13 -1
- data/web/assets/javascripts/application.js +16 -0
- data/web/assets/javascripts/dashboard-charts.js +17 -1
- data/web/assets/javascripts/dashboard.js +7 -9
- data/web/assets/javascripts/metrics.js +34 -0
- data/web/assets/stylesheets/application.css +9 -0
- data/web/locales/en.yml +2 -0
- data/web/locales/pt-br.yml +20 -0
- data/web/views/_job_info.erb +1 -1
- data/web/views/_metrics_period_select.erb +1 -1
- data/web/views/_summary.erb +7 -7
- data/web/views/busy.erb +3 -3
- data/web/views/dashboard.erb +23 -33
- data/web/views/filtering.erb +7 -0
- data/web/views/metrics.erb +36 -27
- data/web/views/metrics_for_job.erb +26 -35
- data/web/views/queues.erb +6 -2
- metadata +4 -3
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 30f824346db9b0ebf8ee13c6ac0101494e5fd6d05b4ed2ef3ad97c5b17ccbc10
|
|
4
|
+
data.tar.gz: bed22f02925116256550bbc34ef9decd70bdbca356d5d61abedd4d14a7dcac45
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 347e82cf6f215a1e4bd09c3f12d382be79d96bb83ccd1d09f928db19b936c2dd72c0f2e3a8988160086da7928a7911d84eddd005428c7b6a337763c4b60a3492
|
|
7
|
+
data.tar.gz: ef0a03f45d4d35e832f36b7a09ae0694838cd0cf3582dc20d53956bb5a61ba07811d410600b43a9bdf777e68dd2549910a125d23885afca8388b8513bb7ac8d1
|
data/Changes.md
CHANGED
|
@@ -2,6 +2,85 @@
|
|
|
2
2
|
|
|
3
3
|
[Sidekiq Changes](https://github.com/sidekiq/sidekiq/blob/main/Changes.md) | [Sidekiq Pro Changes](https://github.com/sidekiq/sidekiq/blob/main/Pro-Changes.md) | [Sidekiq Enterprise Changes](https://github.com/sidekiq/sidekiq/blob/main/Ent-Changes.md)
|
|
4
4
|
|
|
5
|
+
7.2.0
|
|
6
|
+
----------
|
|
7
|
+
|
|
8
|
+
- `sidekiq_retries_exhausted` can return `:discard` to avoid the deadset
|
|
9
|
+
and all death handlers [#6091]
|
|
10
|
+
- Metrics filtering by job class in Web UI [#5974]
|
|
11
|
+
- Better readability and formatting for numbers within the Web UI [#6080]
|
|
12
|
+
- Add explicit error if user code tries to nest test modes [#6078]
|
|
13
|
+
```ruby
|
|
14
|
+
Sidekiq::Testing.inline! # global setting
|
|
15
|
+
Sidekiq::Testing.fake! do # override within block
|
|
16
|
+
# ok
|
|
17
|
+
Sidekiq::Testing.inline! do # can't override the override
|
|
18
|
+
# not ok, nested
|
|
19
|
+
end
|
|
20
|
+
end
|
|
21
|
+
```
|
|
22
|
+
- **SECURITY** Forbid inline JavaScript execution in Web UI [#6074]
|
|
23
|
+
- Adjust redis-client adapter to avoid `method_missing` [#6083]
|
|
24
|
+
This can result in app code breaking if your app's Redis API usage was
|
|
25
|
+
depending on Sidekiq's adapter to correct invalid redis-client API usage.
|
|
26
|
+
One example:
|
|
27
|
+
```ruby
|
|
28
|
+
# bad, not redis-client native
|
|
29
|
+
# Unsupported command argument type: TrueClass (TypeError)
|
|
30
|
+
Sidekiq.redis { |c| c.set("key", "value", nx: true, ex: 15) }
|
|
31
|
+
# good
|
|
32
|
+
Sidekiq.redis { |c| c.set("key", "value", "nx", "ex", 15) }
|
|
33
|
+
```
|
|
34
|
+
|
|
35
|
+
7.1.6
|
|
36
|
+
----------
|
|
37
|
+
|
|
38
|
+
- The block forms of testing modes (inline, fake) are now thread-safe so you can have
|
|
39
|
+
a multithreaded test suite which uses different modes for different tests. [#6069]
|
|
40
|
+
- Fix breakage with non-Proc error handlers [#6065]
|
|
41
|
+
|
|
42
|
+
7.1.5
|
|
43
|
+
----------
|
|
44
|
+
|
|
45
|
+
- **FEATURE**: Job filtering within the Web UI. This feature has been open
|
|
46
|
+
sourced from Sidekiq Pro. [#6052]
|
|
47
|
+
- **API CHANGE** Error handlers now take three arguments `->(ex, context, config)`.
|
|
48
|
+
The previous calling convention will work until Sidekiq 8.0 but will print
|
|
49
|
+
out a deprecation warning. [#6051]
|
|
50
|
+
- Fix issue with the `batch_size` and `at` options in `S::Client.push_bulk` [#6040]
|
|
51
|
+
- Fix inline testing firing batch callbacks early [#6057]
|
|
52
|
+
- Use new log broadcast API in Rails 7.1 [#6054]
|
|
53
|
+
- Crash if user tries to use RESP2 `protocol: 2` [#6061]
|
|
54
|
+
|
|
55
|
+
7.1.4
|
|
56
|
+
----------
|
|
57
|
+
|
|
58
|
+
- Fix empty `retry_for` logic [#6035]
|
|
59
|
+
|
|
60
|
+
7.1.3
|
|
61
|
+
----------
|
|
62
|
+
|
|
63
|
+
- Add `sidekiq_options retry_for: 48.hours` to allow time-based retry windows [#6029]
|
|
64
|
+
- Support sidekiq_retry_in and sidekiq_retries_exhausted_block in ActiveJobs (#5994)
|
|
65
|
+
- Lowercase all Rack headers for Rack 3.0 [#5951]
|
|
66
|
+
- Validate Sidekiq::Web page refresh delay to avoid potential DoS,
|
|
67
|
+
CVE-2023-26141, thanks for reporting Keegan!
|
|
68
|
+
|
|
69
|
+
7.1.2
|
|
70
|
+
----------
|
|
71
|
+
|
|
72
|
+
- Mark Web UI assets as private so CDNs won't cache them [#5936]
|
|
73
|
+
- Fix stackoverflow when using Oj and the JSON log formatter [#5920]
|
|
74
|
+
- Remove spurious `enqueued_at` from scheduled ActiveJobs [#5937]
|
|
75
|
+
|
|
76
|
+
7.1.1
|
|
77
|
+
----------
|
|
78
|
+
|
|
79
|
+
- Support multiple CurrentAttributes [#5904]
|
|
80
|
+
- Speed up latency fetch with large queues on Redis <7 [#5910]
|
|
81
|
+
- Allow a larger default client pool [#5886]
|
|
82
|
+
- Ensure Sidekiq.options[:environment] == RAILS_ENV [#5932]
|
|
83
|
+
|
|
5
84
|
7.1.0
|
|
6
85
|
----------
|
|
7
86
|
|
|
@@ -101,6 +180,16 @@ end
|
|
|
101
180
|
- Job Execution metrics!!!
|
|
102
181
|
- See `docs/7.0-Upgrade.md` for release notes
|
|
103
182
|
|
|
183
|
+
6.5.{10,11,12}
|
|
184
|
+
----------
|
|
185
|
+
|
|
186
|
+
- Fixes for Rails 7.1 [#6067, #6070]
|
|
187
|
+
|
|
188
|
+
6.5.9
|
|
189
|
+
----------
|
|
190
|
+
|
|
191
|
+
- Ensure Sidekiq.options[:environment] == RAILS_ENV [#5932]
|
|
192
|
+
|
|
104
193
|
6.5.8
|
|
105
194
|
----------
|
|
106
195
|
|
data/README.md
CHANGED
|
@@ -83,7 +83,7 @@ You can purchase at https://sidekiq.org; email support@contribsys.com for help.
|
|
|
83
83
|
Useful resources:
|
|
84
84
|
|
|
85
85
|
* Product documentation is in the [wiki](https://github.com/sidekiq/sidekiq/wiki).
|
|
86
|
-
* Occasional announcements are made to the [@sidekiq](https://
|
|
86
|
+
* Occasional announcements are made to the [@sidekiq](https://ruby.social/@sidekiq) Mastodon account.
|
|
87
87
|
* The [Sidekiq tag](https://stackoverflow.com/questions/tagged/sidekiq) on Stack Overflow has lots of useful Q & A.
|
|
88
88
|
|
|
89
89
|
Every Friday morning is Sidekiq office hour: I video chat and answer questions.
|
|
@@ -103,4 +103,4 @@ The license for Sidekiq Pro and Sidekiq Enterprise can be found in [COMM-LICENSE
|
|
|
103
103
|
Author
|
|
104
104
|
-----------------
|
|
105
105
|
|
|
106
|
-
Mike Perham, [@getajobmike](https://
|
|
106
|
+
Mike Perham, [@getajobmike](https://ruby.social/@getajobmike) / [@sidekiq](https://ruby.social/@sidekiq), [https://www.mikeperham.com](https://www.mikeperham.com) / [https://www.contribsys.com](https://www.contribsys.com)
|
data/lib/sidekiq/api.rb
CHANGED
|
@@ -92,11 +92,11 @@ module Sidekiq
|
|
|
92
92
|
pipeline.zcard("retry")
|
|
93
93
|
pipeline.zcard("dead")
|
|
94
94
|
pipeline.scard("processes")
|
|
95
|
-
pipeline.
|
|
95
|
+
pipeline.lindex("queue:default", -1)
|
|
96
96
|
end
|
|
97
97
|
}
|
|
98
98
|
|
|
99
|
-
default_queue_latency = if (entry = pipe1_res[6]
|
|
99
|
+
default_queue_latency = if (entry = pipe1_res[6])
|
|
100
100
|
job = begin
|
|
101
101
|
Sidekiq.load_json(entry)
|
|
102
102
|
rescue
|
|
@@ -264,8 +264,8 @@ module Sidekiq
|
|
|
264
264
|
# @return [Float] in seconds
|
|
265
265
|
def latency
|
|
266
266
|
entry = Sidekiq.redis { |conn|
|
|
267
|
-
conn.
|
|
268
|
-
}
|
|
267
|
+
conn.lindex(@rname, -1)
|
|
268
|
+
}
|
|
269
269
|
return 0 unless entry
|
|
270
270
|
job = Sidekiq.load_json(entry)
|
|
271
271
|
now = Time.now.to_f
|
|
@@ -679,7 +679,7 @@ module Sidekiq
|
|
|
679
679
|
range_start = page * page_size + offset_size
|
|
680
680
|
range_end = range_start + page_size - 1
|
|
681
681
|
elements = Sidekiq.redis { |conn|
|
|
682
|
-
conn.zrange name, range_start, range_end, withscores
|
|
682
|
+
conn.zrange name, range_start, range_end, "withscores"
|
|
683
683
|
}
|
|
684
684
|
break if elements.empty?
|
|
685
685
|
page -= 1
|
|
@@ -706,7 +706,7 @@ module Sidekiq
|
|
|
706
706
|
end
|
|
707
707
|
|
|
708
708
|
elements = Sidekiq.redis { |conn|
|
|
709
|
-
conn.zrange(name, begin_score, end_score, "BYSCORE", withscores
|
|
709
|
+
conn.zrange(name, begin_score, end_score, "BYSCORE", "withscores")
|
|
710
710
|
}
|
|
711
711
|
|
|
712
712
|
elements.each_with_object([]) do |element, result|
|
|
@@ -881,7 +881,7 @@ module Sidekiq
|
|
|
881
881
|
# @api private
|
|
882
882
|
def cleanup
|
|
883
883
|
# dont run cleanup more than once per minute
|
|
884
|
-
return 0 unless Sidekiq.redis { |conn| conn.set("process_cleanup", "1",
|
|
884
|
+
return 0 unless Sidekiq.redis { |conn| conn.set("process_cleanup", "1", "NX", "EX", "60") }
|
|
885
885
|
|
|
886
886
|
count = 0
|
|
887
887
|
Sidekiq.redis do |conn|
|
data/lib/sidekiq/cli.rb
CHANGED
|
@@ -230,6 +230,7 @@ module Sidekiq # :nodoc:
|
|
|
230
230
|
# Both Sinatra 2.0+ and Sidekiq support this term.
|
|
231
231
|
# RAILS_ENV and RACK_ENV are there for legacy support.
|
|
232
232
|
@environment = cli_env || ENV["APP_ENV"] || ENV["RAILS_ENV"] || ENV["RACK_ENV"] || "development"
|
|
233
|
+
config[:environment] = @environment
|
|
233
234
|
end
|
|
234
235
|
|
|
235
236
|
def symbolize_keys_deep!(hash)
|
data/lib/sidekiq/client.rb
CHANGED
|
@@ -66,6 +66,7 @@ module Sidekiq
|
|
|
66
66
|
# args - an array of simple arguments to the perform method, must be JSON-serializable
|
|
67
67
|
# at - timestamp to schedule the job (optional), must be Numeric (e.g. Time.now.to_f)
|
|
68
68
|
# retry - whether to retry this job if it fails, default true or an integer number of retries
|
|
69
|
+
# retry_for - relative amount of time to retry this job if it fails, default nil
|
|
69
70
|
# backtrace - whether to save any error backtrace, default false
|
|
70
71
|
#
|
|
71
72
|
# If class is set to the class name, the jobs' options will be based on Sidekiq's default
|
|
@@ -73,7 +74,7 @@ module Sidekiq
|
|
|
73
74
|
#
|
|
74
75
|
# Any options valid for a job class's sidekiq_options are also available here.
|
|
75
76
|
#
|
|
76
|
-
# All
|
|
77
|
+
# All keys must be strings, not symbols. NB: because we are serializing to JSON, all
|
|
77
78
|
# symbols in 'args' will be converted to strings. Note that +backtrace: true+ can take quite a bit of
|
|
78
79
|
# space in Redis; a large volume of failing jobs can start Redis swapping if you aren't careful.
|
|
79
80
|
#
|
|
@@ -110,7 +111,7 @@ module Sidekiq
|
|
|
110
111
|
# prevented a job push.
|
|
111
112
|
#
|
|
112
113
|
# Example (pushing jobs in batches):
|
|
113
|
-
# push_bulk('class' =>
|
|
114
|
+
# push_bulk('class' => MyJob, 'args' => (1..100_000).to_a, batch_size: 1_000)
|
|
114
115
|
#
|
|
115
116
|
def push_bulk(items)
|
|
116
117
|
batch_size = items.delete(:batch_size) || items.delete("batch_size") || 1_000
|
|
@@ -123,19 +124,21 @@ module Sidekiq
|
|
|
123
124
|
raise ArgumentError, "Explicitly passing 'jid' when pushing more than one job is not supported" if jid && args.size > 1
|
|
124
125
|
|
|
125
126
|
normed = normalize_item(items)
|
|
127
|
+
slice_index = 0
|
|
126
128
|
result = args.each_slice(batch_size).flat_map do |slice|
|
|
127
129
|
raise ArgumentError, "Bulk arguments must be an Array of Arrays: [[1], [2]]" unless slice.is_a?(Array) && slice.all?(Array)
|
|
128
130
|
break [] if slice.empty? # no jobs to push
|
|
129
131
|
|
|
130
132
|
payloads = slice.map.with_index { |job_args, index|
|
|
131
133
|
copy = normed.merge("args" => job_args, "jid" => SecureRandom.hex(12))
|
|
132
|
-
copy["at"] = (at.is_a?(Array) ? at[index] : at) if at
|
|
134
|
+
copy["at"] = (at.is_a?(Array) ? at[slice_index + index] : at) if at
|
|
133
135
|
result = middleware.invoke(items["class"], copy, copy["queue"], @redis_pool) do
|
|
134
136
|
verify_json(copy)
|
|
135
137
|
copy
|
|
136
138
|
end
|
|
137
139
|
result || nil
|
|
138
140
|
}
|
|
141
|
+
slice_index += batch_size
|
|
139
142
|
|
|
140
143
|
to_push = payloads.compact
|
|
141
144
|
raw_push(to_push) unless to_push.empty?
|
|
@@ -246,6 +249,8 @@ module Sidekiq
|
|
|
246
249
|
if payloads.first.key?("at")
|
|
247
250
|
conn.zadd("schedule", payloads.flat_map { |hash|
|
|
248
251
|
at = hash.delete("at").to_s
|
|
252
|
+
# ActiveJob sets this but the job has not been enqueued yet
|
|
253
|
+
hash.delete("enqueued_at")
|
|
249
254
|
[at, Sidekiq.dump_json(hash)]
|
|
250
255
|
})
|
|
251
256
|
else
|
data/lib/sidekiq/config.rb
CHANGED
|
@@ -34,8 +34,7 @@ module Sidekiq
|
|
|
34
34
|
backtrace_cleaner: ->(backtrace) { backtrace }
|
|
35
35
|
}
|
|
36
36
|
|
|
37
|
-
ERROR_HANDLER = ->(ex, ctx) {
|
|
38
|
-
cfg = ctx[:_config] || Sidekiq.default_configuration
|
|
37
|
+
ERROR_HANDLER = ->(ex, ctx, cfg = Sidekiq.default_configuration) {
|
|
39
38
|
l = cfg.logger
|
|
40
39
|
l.warn(Sidekiq.dump_json(ctx)) unless ctx.empty?
|
|
41
40
|
l.warn("#{ex.class.name}: #{ex.message}")
|
|
@@ -56,6 +55,10 @@ module Sidekiq
|
|
|
56
55
|
def_delegators :@options, :[], :[]=, :fetch, :key?, :has_key?, :merge!
|
|
57
56
|
attr_reader :capsules
|
|
58
57
|
|
|
58
|
+
def to_json(*)
|
|
59
|
+
Sidekiq.dump_json(@options)
|
|
60
|
+
end
|
|
61
|
+
|
|
59
62
|
# LEGACY: edits the default capsule
|
|
60
63
|
# config.concurrency = 5
|
|
61
64
|
def concurrency=(val)
|
|
@@ -127,7 +130,7 @@ module Sidekiq
|
|
|
127
130
|
private def local_redis_pool
|
|
128
131
|
# this is our internal client/housekeeping pool. each capsule has its
|
|
129
132
|
# own pool for executing threads.
|
|
130
|
-
@redis ||= new_redis_pool(
|
|
133
|
+
@redis ||= new_redis_pool(10, "internal")
|
|
131
134
|
end
|
|
132
135
|
|
|
133
136
|
def new_redis_pool(size, name = "unset")
|
|
@@ -255,15 +258,25 @@ module Sidekiq
|
|
|
255
258
|
@logger = logger
|
|
256
259
|
end
|
|
257
260
|
|
|
261
|
+
private def parameter_size(handler)
|
|
262
|
+
target = handler.is_a?(Proc) ? handler : handler.method(:call)
|
|
263
|
+
target.parameters.size
|
|
264
|
+
end
|
|
265
|
+
|
|
258
266
|
# INTERNAL USE ONLY
|
|
259
267
|
def handle_exception(ex, ctx = {})
|
|
260
268
|
if @options[:error_handlers].size == 0
|
|
261
269
|
p ["!!!!!", ex]
|
|
262
270
|
end
|
|
263
|
-
ctx[:_config] = self
|
|
264
271
|
@options[:error_handlers].each do |handler|
|
|
265
|
-
handler
|
|
266
|
-
|
|
272
|
+
if parameter_size(handler) == 2
|
|
273
|
+
# TODO Remove in 8.0
|
|
274
|
+
logger.info { "DEPRECATION: Sidekiq exception handlers now take three arguments, see #{handler}" }
|
|
275
|
+
handler.call(ex, {_config: self}.merge(ctx))
|
|
276
|
+
else
|
|
277
|
+
handler.call(ex, ctx, self)
|
|
278
|
+
end
|
|
279
|
+
rescue Exception => e
|
|
267
280
|
l = logger
|
|
268
281
|
l.error "!!! ERROR HANDLER THREW AN ERROR !!!"
|
|
269
282
|
l.error e
|
data/lib/sidekiq/deploy.rb
CHANGED
|
@@ -44,7 +44,7 @@ module Sidekiq
|
|
|
44
44
|
|
|
45
45
|
@pool.with do |c|
|
|
46
46
|
# only allow one deploy mark for a given label for the next minute
|
|
47
|
-
lock = c.set("deploylock-#{label}", stamp, nx
|
|
47
|
+
lock = c.set("deploylock-#{label}", stamp, "nx", "ex", "60")
|
|
48
48
|
if lock
|
|
49
49
|
c.multi do |pipe|
|
|
50
50
|
pipe.hsetnx(key, stamp, label)
|
data/lib/sidekiq/job_retry.rb
CHANGED
|
@@ -170,9 +170,11 @@ module Sidekiq
|
|
|
170
170
|
msg["error_backtrace"] = compress_backtrace(lines)
|
|
171
171
|
end
|
|
172
172
|
|
|
173
|
-
# Goodbye dear message, you (re)tried your best I'm sure.
|
|
174
173
|
return retries_exhausted(jobinst, msg, exception) if count >= max_retry_attempts
|
|
175
174
|
|
|
175
|
+
rf = msg["retry_for"]
|
|
176
|
+
return retries_exhausted(jobinst, msg, exception) if rf && ((msg["failed_at"] + rf) < Time.now.to_f)
|
|
177
|
+
|
|
176
178
|
strategy, delay = delay_for(jobinst, count, exception, msg)
|
|
177
179
|
case strategy
|
|
178
180
|
when :discard
|
|
@@ -197,7 +199,14 @@ module Sidekiq
|
|
|
197
199
|
# sidekiq_retry_in can return two different things:
|
|
198
200
|
# 1. When to retry next, as an integer of seconds
|
|
199
201
|
# 2. A symbol which re-routes the job elsewhere, e.g. :discard, :kill, :default
|
|
200
|
-
jobinst&.sidekiq_retry_in_block
|
|
202
|
+
block = jobinst&.sidekiq_retry_in_block
|
|
203
|
+
|
|
204
|
+
# the sidekiq_retry_in_block can be defined in a wrapped class (ActiveJob for instance)
|
|
205
|
+
unless msg["wrapped"].nil?
|
|
206
|
+
wrapped = Object.const_get(msg["wrapped"])
|
|
207
|
+
block = wrapped.respond_to?(:sidekiq_retry_in_block) ? wrapped.sidekiq_retry_in_block : nil
|
|
208
|
+
end
|
|
209
|
+
block&.call(count, exception, msg)
|
|
201
210
|
rescue Exception => e
|
|
202
211
|
handle_exception(e, {context: "Failure scheduling retry using the defined `sidekiq_retry_in` in #{jobinst.class.name}, falling back to default"})
|
|
203
212
|
nil
|
|
@@ -217,13 +226,20 @@ module Sidekiq
|
|
|
217
226
|
end
|
|
218
227
|
|
|
219
228
|
def retries_exhausted(jobinst, msg, exception)
|
|
220
|
-
begin
|
|
229
|
+
rv = begin
|
|
221
230
|
block = jobinst&.sidekiq_retries_exhausted_block
|
|
231
|
+
|
|
232
|
+
# the sidekiq_retries_exhausted_block can be defined in a wrapped class (ActiveJob for instance)
|
|
233
|
+
unless msg["wrapped"].nil?
|
|
234
|
+
wrapped = Object.const_get(msg["wrapped"])
|
|
235
|
+
block = wrapped.respond_to?(:sidekiq_retries_exhausted_block) ? wrapped.sidekiq_retries_exhausted_block : nil
|
|
236
|
+
end
|
|
222
237
|
block&.call(msg, exception)
|
|
223
238
|
rescue => e
|
|
224
239
|
handle_exception(e, {context: "Error calling retries_exhausted", job: msg})
|
|
225
240
|
end
|
|
226
241
|
|
|
242
|
+
return if rv == :discard # poof!
|
|
227
243
|
send_to_morgue(msg) unless msg["dead"] == false
|
|
228
244
|
|
|
229
245
|
@capsule.config.death_handlers.each do |handler|
|
data/lib/sidekiq/job_util.rb
CHANGED
|
@@ -13,6 +13,7 @@ module Sidekiq
|
|
|
13
13
|
raise(ArgumentError, "Job class must be either a Class or String representation of the class name: `#{item}`") unless item["class"].is_a?(Class) || item["class"].is_a?(String)
|
|
14
14
|
raise(ArgumentError, "Job 'at' must be a Numeric timestamp: `#{item}`") if item.key?("at") && !item["at"].is_a?(Numeric)
|
|
15
15
|
raise(ArgumentError, "Job tags must be an Array: `#{item}`") if item["tags"] && !item["tags"].is_a?(Array)
|
|
16
|
+
raise(ArgumentError, "retry_for must be a relative amount of time, e.g. 48.hours `#{item}`") if item["retry_for"] && item["retry_for"] > 1_000_000_000
|
|
16
17
|
end
|
|
17
18
|
|
|
18
19
|
def verify_json(item)
|
|
@@ -54,6 +55,7 @@ module Sidekiq
|
|
|
54
55
|
item["jid"] ||= SecureRandom.hex(12)
|
|
55
56
|
item["class"] = item["class"].to_s
|
|
56
57
|
item["queue"] = item["queue"].to_s
|
|
58
|
+
item["retry_for"] = item["retry_for"].to_i if item["retry_for"]
|
|
57
59
|
item["created_at"] ||= Time.now.to_f
|
|
58
60
|
item
|
|
59
61
|
end
|
|
@@ -20,7 +20,8 @@ module Sidekiq
|
|
|
20
20
|
end
|
|
21
21
|
|
|
22
22
|
# Get metric data for all jobs from the last hour
|
|
23
|
-
|
|
23
|
+
# +class_filter+: return only results for classes matching filter
|
|
24
|
+
def top_jobs(class_filter: nil, minutes: 60)
|
|
24
25
|
result = Result.new
|
|
25
26
|
|
|
26
27
|
time = @time
|
|
@@ -39,6 +40,7 @@ module Sidekiq
|
|
|
39
40
|
redis_results.each do |hash|
|
|
40
41
|
hash.each do |k, v|
|
|
41
42
|
kls, metric = k.split("|")
|
|
43
|
+
next if class_filter && !class_filter.match?(kls)
|
|
42
44
|
result.job_results[kls].add_metric metric, time, v.to_i
|
|
43
45
|
end
|
|
44
46
|
time -= 60
|
|
@@ -7,26 +7,32 @@ module Sidekiq
|
|
|
7
7
|
# This can be useful for multi-tenancy, i18n locale, timezone, any implicit
|
|
8
8
|
# per-request attribute. See +ActiveSupport::CurrentAttributes+.
|
|
9
9
|
#
|
|
10
|
+
# For multiple current attributes, pass an array of current attributes.
|
|
11
|
+
#
|
|
10
12
|
# @example
|
|
11
13
|
#
|
|
12
14
|
# # in your initializer
|
|
13
15
|
# require "sidekiq/middleware/current_attributes"
|
|
14
16
|
# Sidekiq::CurrentAttributes.persist("Myapp::Current")
|
|
17
|
+
# # or multiple current attributes
|
|
18
|
+
# Sidekiq::CurrentAttributes.persist(["Myapp::Current", "Myapp::OtherCurrent"])
|
|
15
19
|
#
|
|
16
20
|
module CurrentAttributes
|
|
17
21
|
class Save
|
|
18
22
|
include Sidekiq::ClientMiddleware
|
|
19
23
|
|
|
20
|
-
def initialize(
|
|
21
|
-
@
|
|
24
|
+
def initialize(cattrs)
|
|
25
|
+
@cattrs = cattrs
|
|
22
26
|
end
|
|
23
27
|
|
|
24
28
|
def call(_, job, _, _)
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
29
|
+
@cattrs.each do |(key, strklass)|
|
|
30
|
+
if !job.has_key?(key)
|
|
31
|
+
attrs = strklass.constantize.attributes
|
|
32
|
+
# Retries can push the job N times, we don't
|
|
33
|
+
# want retries to reset cattr. #5692, #5090
|
|
34
|
+
job[key] = attrs if attrs.any?
|
|
35
|
+
end
|
|
30
36
|
end
|
|
31
37
|
yield
|
|
32
38
|
end
|
|
@@ -35,22 +41,55 @@ module Sidekiq
|
|
|
35
41
|
class Load
|
|
36
42
|
include Sidekiq::ServerMiddleware
|
|
37
43
|
|
|
38
|
-
def initialize(
|
|
39
|
-
@
|
|
44
|
+
def initialize(cattrs)
|
|
45
|
+
@cattrs = cattrs
|
|
40
46
|
end
|
|
41
47
|
|
|
42
48
|
def call(_, job, _, &block)
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
49
|
+
cattrs_to_reset = []
|
|
50
|
+
|
|
51
|
+
@cattrs.each do |(key, strklass)|
|
|
52
|
+
if job.has_key?(key)
|
|
53
|
+
constklass = strklass.constantize
|
|
54
|
+
cattrs_to_reset << constklass
|
|
55
|
+
|
|
56
|
+
job[key].each do |(attribute, value)|
|
|
57
|
+
constklass.public_send("#{attribute}=", value)
|
|
58
|
+
end
|
|
59
|
+
end
|
|
47
60
|
end
|
|
61
|
+
|
|
62
|
+
yield
|
|
63
|
+
ensure
|
|
64
|
+
cattrs_to_reset.each(&:reset)
|
|
48
65
|
end
|
|
49
66
|
end
|
|
50
67
|
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
68
|
+
class << self
|
|
69
|
+
def persist(klass_or_array, config = Sidekiq.default_configuration)
|
|
70
|
+
cattrs = build_cattrs_hash(klass_or_array)
|
|
71
|
+
|
|
72
|
+
config.client_middleware.add Save, cattrs
|
|
73
|
+
config.server_middleware.add Load, cattrs
|
|
74
|
+
end
|
|
75
|
+
|
|
76
|
+
private
|
|
77
|
+
|
|
78
|
+
def build_cattrs_hash(klass_or_array)
|
|
79
|
+
if klass_or_array.is_a?(Array)
|
|
80
|
+
{}.tap do |hash|
|
|
81
|
+
klass_or_array.each_with_index do |klass, index|
|
|
82
|
+
hash[key_at(index)] = klass.to_s
|
|
83
|
+
end
|
|
84
|
+
end
|
|
85
|
+
else
|
|
86
|
+
{key_at(0) => klass_or_array.to_s}
|
|
87
|
+
end
|
|
88
|
+
end
|
|
89
|
+
|
|
90
|
+
def key_at(index)
|
|
91
|
+
(index == 0) ? "cattr" : "cattr_#{index}"
|
|
92
|
+
end
|
|
54
93
|
end
|
|
55
94
|
end
|
|
56
95
|
end
|
data/lib/sidekiq/paginator.rb
CHANGED
|
@@ -19,9 +19,9 @@ module Sidekiq
|
|
|
19
19
|
total_size, items = conn.multi { |transaction|
|
|
20
20
|
transaction.zcard(key)
|
|
21
21
|
if rev
|
|
22
|
-
transaction.zrange(key, starting, ending, "REV", withscores
|
|
22
|
+
transaction.zrange(key, starting, ending, "REV", "withscores")
|
|
23
23
|
else
|
|
24
|
-
transaction.zrange(key, starting, ending, withscores
|
|
24
|
+
transaction.zrange(key, starting, ending, "withscores")
|
|
25
25
|
end
|
|
26
26
|
}
|
|
27
27
|
[current_page, total_size, items]
|
data/lib/sidekiq/processor.rb
CHANGED
|
@@ -148,6 +148,8 @@ module Sidekiq
|
|
|
148
148
|
|
|
149
149
|
IGNORE_SHUTDOWN_INTERRUPTS = {Sidekiq::Shutdown => :never}
|
|
150
150
|
private_constant :IGNORE_SHUTDOWN_INTERRUPTS
|
|
151
|
+
ALLOW_SHUTDOWN_INTERRUPTS = {Sidekiq::Shutdown => :immediate}
|
|
152
|
+
private_constant :ALLOW_SHUTDOWN_INTERRUPTS
|
|
151
153
|
|
|
152
154
|
def process(uow)
|
|
153
155
|
jobstr = uow.job
|
|
@@ -171,36 +173,35 @@ module Sidekiq
|
|
|
171
173
|
end
|
|
172
174
|
|
|
173
175
|
ack = false
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
176
|
+
Thread.handle_interrupt(IGNORE_SHUTDOWN_INTERRUPTS) do
|
|
177
|
+
Thread.handle_interrupt(ALLOW_SHUTDOWN_INTERRUPTS) do
|
|
178
|
+
dispatch(job_hash, queue, jobstr) do |inst|
|
|
179
|
+
config.server_middleware.invoke(inst, job_hash, queue) do
|
|
180
|
+
execute_job(inst, job_hash["args"])
|
|
181
|
+
end
|
|
178
182
|
end
|
|
183
|
+
ack = true
|
|
184
|
+
rescue Sidekiq::Shutdown
|
|
185
|
+
# Had to force kill this job because it didn't finish
|
|
186
|
+
# within the timeout. Don't acknowledge the work since
|
|
187
|
+
# we didn't properly finish it.
|
|
188
|
+
rescue Sidekiq::JobRetry::Handled => h
|
|
189
|
+
# this is the common case: job raised error and Sidekiq::JobRetry::Handled
|
|
190
|
+
# signals that we created a retry successfully. We can acknowlege the job.
|
|
191
|
+
ack = true
|
|
192
|
+
e = h.cause || h
|
|
193
|
+
handle_exception(e, {context: "Job raised exception", job: job_hash})
|
|
194
|
+
raise e
|
|
195
|
+
rescue Exception => ex
|
|
196
|
+
# Unexpected error! This is very bad and indicates an exception that got past
|
|
197
|
+
# the retry subsystem (e.g. network partition). We won't acknowledge the job
|
|
198
|
+
# so it can be rescued when using Sidekiq Pro.
|
|
199
|
+
handle_exception(ex, {context: "Internal exception!", job: job_hash, jobstr: jobstr})
|
|
200
|
+
raise ex
|
|
179
201
|
end
|
|
180
|
-
ack = true
|
|
181
|
-
rescue Sidekiq::Shutdown
|
|
182
|
-
# Had to force kill this job because it didn't finish
|
|
183
|
-
# within the timeout. Don't acknowledge the work since
|
|
184
|
-
# we didn't properly finish it.
|
|
185
|
-
rescue Sidekiq::JobRetry::Handled => h
|
|
186
|
-
# this is the common case: job raised error and Sidekiq::JobRetry::Handled
|
|
187
|
-
# signals that we created a retry successfully. We can acknowlege the job.
|
|
188
|
-
ack = true
|
|
189
|
-
e = h.cause || h
|
|
190
|
-
handle_exception(e, {context: "Job raised exception", job: job_hash})
|
|
191
|
-
raise e
|
|
192
|
-
rescue Exception => ex
|
|
193
|
-
# Unexpected error! This is very bad and indicates an exception that got past
|
|
194
|
-
# the retry subsystem (e.g. network partition). We won't acknowledge the job
|
|
195
|
-
# so it can be rescued when using Sidekiq Pro.
|
|
196
|
-
handle_exception(ex, {context: "Internal exception!", job: job_hash, jobstr: jobstr})
|
|
197
|
-
raise ex
|
|
198
202
|
ensure
|
|
199
203
|
if ack
|
|
200
|
-
|
|
201
|
-
Thread.handle_interrupt(IGNORE_SHUTDOWN_INTERRUPTS) do
|
|
202
|
-
uow.acknowledge
|
|
203
|
-
end
|
|
204
|
+
uow.acknowledge
|
|
204
205
|
end
|
|
205
206
|
end
|
|
206
207
|
end
|
data/lib/sidekiq/rails.rb
CHANGED
|
@@ -39,17 +39,6 @@ module Sidekiq
|
|
|
39
39
|
end
|
|
40
40
|
end
|
|
41
41
|
|
|
42
|
-
initializer "sidekiq.rails_logger" do
|
|
43
|
-
Sidekiq.configure_server do |config|
|
|
44
|
-
# This is the integration code necessary so that if a job uses `Rails.logger.info "Hello"`,
|
|
45
|
-
# it will appear in the Sidekiq console with all of the job context. See #5021 and
|
|
46
|
-
# https://github.com/rails/rails/blob/b5f2b550f69a99336482739000c58e4e04e033aa/railties/lib/rails/commands/server/server_command.rb#L82-L84
|
|
47
|
-
unless ::Rails.logger == config.logger || ::ActiveSupport::Logger.logger_outputs_to?(::Rails.logger, $stdout)
|
|
48
|
-
::Rails.logger.extend(::ActiveSupport::Logger.broadcast(config.logger))
|
|
49
|
-
end
|
|
50
|
-
end
|
|
51
|
-
end
|
|
52
|
-
|
|
53
42
|
initializer "sidekiq.backtrace_cleaner" do
|
|
54
43
|
Sidekiq.configure_server do |config|
|
|
55
44
|
config[:backtrace_cleaner] = ->(backtrace) { ::Rails.backtrace_cleaner.clean(backtrace) }
|
|
@@ -63,6 +52,16 @@ module Sidekiq
|
|
|
63
52
|
config.after_initialize do
|
|
64
53
|
Sidekiq.configure_server do |config|
|
|
65
54
|
config[:reloader] = Sidekiq::Rails::Reloader.new
|
|
55
|
+
|
|
56
|
+
# This is the integration code necessary so that if a job uses `Rails.logger.info "Hello"`,
|
|
57
|
+
# it will appear in the Sidekiq console with all of the job context.
|
|
58
|
+
unless ::Rails.logger == config.logger || ::ActiveSupport::Logger.logger_outputs_to?(::Rails.logger, $stdout)
|
|
59
|
+
if ::Rails::VERSION::STRING < "7.1"
|
|
60
|
+
::Rails.logger.extend(::ActiveSupport::Logger.broadcast(config.logger))
|
|
61
|
+
else
|
|
62
|
+
::Rails.logger.broadcast_to(config.logger)
|
|
63
|
+
end
|
|
64
|
+
end
|
|
66
65
|
end
|
|
67
66
|
end
|
|
68
67
|
end
|