sidekiq 7.2.1 → 7.2.3
Sign up to get free protection for your applications and to get access to all the features.
Potentially problematic release.
This version of sidekiq might be problematic. Click here for more details.
- checksums.yaml +4 -4
- data/Changes.md +14 -0
- data/README.md +2 -2
- data/bin/multi_queue_bench +41 -38
- data/lib/sidekiq/api.rb +18 -4
- data/lib/sidekiq/cli.rb +3 -1
- data/lib/sidekiq/deploy.rb +1 -1
- data/lib/sidekiq/job.rb +1 -1
- data/lib/sidekiq/launcher.rb +6 -4
- data/lib/sidekiq/logger.rb +1 -1
- data/lib/sidekiq/middleware/current_attributes.rb +1 -1
- data/lib/sidekiq/processor.rb +1 -1
- data/lib/sidekiq/rails.rb +4 -0
- data/lib/sidekiq/redis_client_adapter.rb +2 -2
- data/lib/sidekiq/redis_connection.rb +1 -3
- data/lib/sidekiq/scheduled.rb +1 -1
- data/lib/sidekiq/testing.rb +1 -1
- data/lib/sidekiq/transaction_aware_client.rb +7 -0
- data/lib/sidekiq/version.rb +1 -1
- data/lib/sidekiq/web/action.rb +5 -0
- data/lib/sidekiq/web/application.rb +15 -3
- data/lib/sidekiq/web/csrf_protection.rb +2 -2
- data/lib/sidekiq/web/helpers.rb +6 -1
- data/web/assets/javascripts/application.js +6 -0
- data/web/assets/stylesheets/application-rtl.css +10 -0
- data/web/assets/stylesheets/application.css +13 -0
- data/web/views/_footer.erb +13 -1
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: e20a9134fa2b226bd69fb52fef9e831efa3c598e95f17f1edcd7c5c765fdee0d
|
4
|
+
data.tar.gz: d9b6a8e217c753b67d7f04ef9656ac81aaa767c92818c7b3ff0d6078901f7151
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: e88f678b545310eada86b4df6eee4b3b2a479ae59743faaf73ddeb3455738f5be80a9c5b3a3a8c4abbe7ad775c1bd928be6dca518cce4cea8e7a3a5aa6c9a199
|
7
|
+
data.tar.gz: 46ae598a14f1c46e5f0baabfd5a6a2f16516028340ff21a1a75e90f4e0dab8d366db9bef563f90ea15351794bd5d93f03baa74e62bfd18c42eca5594002f6b7d
|
data/Changes.md
CHANGED
@@ -2,6 +2,20 @@
|
|
2
2
|
|
3
3
|
[Sidekiq Changes](https://github.com/sidekiq/sidekiq/blob/main/Changes.md) | [Sidekiq Pro Changes](https://github.com/sidekiq/sidekiq/blob/main/Pro-Changes.md) | [Sidekiq Enterprise Changes](https://github.com/sidekiq/sidekiq/blob/main/Ent-Changes.md)
|
4
4
|
|
5
|
+
7.2.3
|
6
|
+
----------
|
7
|
+
|
8
|
+
- [Support Dragonfly.io](https://www.mikeperham.com/2024/02/01/supporting-dragonfly/) as an alternative Redis implementation
|
9
|
+
- Fix error unpacking some compressed error backtraces [#6241]
|
10
|
+
- Fix potential heartbeat data leak [#6227]
|
11
|
+
- Add ability to find a currently running work by jid [#6212, fatkodima]
|
12
|
+
|
13
|
+
7.2.2
|
14
|
+
----------
|
15
|
+
|
16
|
+
- Add `Process.warmup` call in Ruby 3.3+
|
17
|
+
- Batch jobs now skip transactional push [#6160]
|
18
|
+
|
5
19
|
7.2.1
|
6
20
|
----------
|
7
21
|
|
data/README.md
CHANGED
@@ -14,11 +14,11 @@ Rails to make background processing dead simple.
|
|
14
14
|
Requirements
|
15
15
|
-----------------
|
16
16
|
|
17
|
-
- Redis: 6.2+
|
17
|
+
- Redis: Redis 6.2+ or Dragonfly 1.13+
|
18
18
|
- Ruby: MRI 2.7+ or JRuby 9.3+.
|
19
19
|
|
20
20
|
Sidekiq 7.0 supports Rails 6.0+ but does not require it.
|
21
|
-
|
21
|
+
As of 7.2, Sidekiq supports Dragonfly as an alternative to Redis for data storage.
|
22
22
|
|
23
23
|
Installation
|
24
24
|
-----------------
|
data/bin/multi_queue_bench
CHANGED
@@ -120,9 +120,12 @@ class Loader
|
|
120
120
|
start = Time.now
|
121
121
|
@iter.times do
|
122
122
|
arr = Array.new(@count) { |idx| [idx] }
|
123
|
-
#always prepends
|
123
|
+
# Sidekiq always prepends "queue:" to the queue name,
|
124
|
+
# that's why we pass 'q1', 'q2', etc instead of 'queue:q1'
|
124
125
|
Sidekiq::Client.push_bulk("class" => LoadWorker, "args" => arr, "queue" => queue)
|
126
|
+
$stdout.write "."
|
125
127
|
end
|
128
|
+
puts "Done"
|
126
129
|
end
|
127
130
|
|
128
131
|
def monitor_single(queue)
|
@@ -176,7 +179,7 @@ class Loader
|
|
176
179
|
end
|
177
180
|
|
178
181
|
def run(queues, queue, monitor_all_queues)
|
179
|
-
|
182
|
+
Sidekiq.logger.warn("Consuming from #{queue}")
|
180
183
|
if monitor_all_queues
|
181
184
|
monitor_all(queues)
|
182
185
|
else
|
@@ -216,50 +219,50 @@ end
|
|
216
219
|
|
217
220
|
# We assign one queue to each sidekiq process
|
218
221
|
def run(number_of_processes, total_queues)
|
219
|
-
|
222
|
+
read_stream, write_stream = IO.pipe
|
220
223
|
|
221
|
-
|
222
|
-
|
223
|
-
|
224
|
-
|
224
|
+
queues = []
|
225
|
+
(0..total_queues-1).each do |idx|
|
226
|
+
queues.push("queue:q#{idx}")
|
227
|
+
end
|
225
228
|
|
226
|
-
|
229
|
+
Sidekiq.logger.info("Queues are: #{queues}")
|
227
230
|
|
228
|
-
|
229
|
-
|
230
|
-
|
231
|
-
|
232
|
-
|
233
|
-
|
234
|
-
end
|
231
|
+
# Produce
|
232
|
+
start = Time.now
|
233
|
+
(0..total_queues-1).each do |idx|
|
234
|
+
Process.fork do
|
235
|
+
queue_num = "q#{idx}"
|
236
|
+
setup(queue_num)
|
235
237
|
end
|
238
|
+
end
|
236
239
|
|
237
|
-
|
238
|
-
|
239
|
-
|
240
|
-
|
241
|
-
|
242
|
-
|
243
|
-
|
244
|
-
|
245
|
-
|
246
|
-
|
247
|
-
|
248
|
-
|
249
|
-
|
250
|
-
|
251
|
-
|
252
|
-
|
253
|
-
|
254
|
-
|
255
|
-
end
|
240
|
+
queue_sz = $iterations * $elements * total_queues
|
241
|
+
Process.waitall
|
242
|
+
|
243
|
+
ending = Time.now - start
|
244
|
+
#Sidekiq.logger.info("Pushed #{queue_sz} in #{ending} secs")
|
245
|
+
|
246
|
+
# Consume
|
247
|
+
(0..number_of_processes-1).each do |idx|
|
248
|
+
Process.fork do
|
249
|
+
# First process only consumes from it's own queue but monitors all queues.
|
250
|
+
# It works as a synchronization point. Once all processes finish
|
251
|
+
# (that is, when all queues are emptied) it prints the the stats.
|
252
|
+
if idx == 0
|
253
|
+
queue = "q#{idx}"
|
254
|
+
consume(queues, queue, true)
|
255
|
+
else
|
256
|
+
queue = "q#{idx % total_queues}"
|
257
|
+
consume(queues, queue, false)
|
256
258
|
end
|
257
259
|
end
|
260
|
+
end
|
258
261
|
|
259
|
-
|
260
|
-
|
261
|
-
|
262
|
-
|
262
|
+
Process.waitall
|
263
|
+
write_stream.close
|
264
|
+
results = read_stream.read
|
265
|
+
read_stream.close
|
263
266
|
end
|
264
267
|
|
265
268
|
$total_processes = ENV["PROCESSES"] ? Integer(ENV["PROCESSES"]) : 8;
|
data/lib/sidekiq/api.rb
CHANGED
@@ -490,7 +490,7 @@ module Sidekiq
|
|
490
490
|
end
|
491
491
|
|
492
492
|
def uncompress_backtrace(backtrace)
|
493
|
-
strict_base64_decoded = backtrace.unpack1("
|
493
|
+
strict_base64_decoded = backtrace.unpack1("m")
|
494
494
|
uncompressed = Zlib::Inflate.inflate(strict_base64_decoded)
|
495
495
|
Sidekiq.load_json(uncompressed)
|
496
496
|
end
|
@@ -773,7 +773,7 @@ module Sidekiq
|
|
773
773
|
#
|
774
774
|
class ScheduledSet < JobSet
|
775
775
|
def initialize
|
776
|
-
super
|
776
|
+
super("schedule")
|
777
777
|
end
|
778
778
|
end
|
779
779
|
|
@@ -787,7 +787,7 @@ module Sidekiq
|
|
787
787
|
#
|
788
788
|
class RetrySet < JobSet
|
789
789
|
def initialize
|
790
|
-
super
|
790
|
+
super("retry")
|
791
791
|
end
|
792
792
|
|
793
793
|
# Enqueues all jobs pending within the retry set.
|
@@ -808,7 +808,7 @@ module Sidekiq
|
|
808
808
|
#
|
809
809
|
class DeadSet < JobSet
|
810
810
|
def initialize
|
811
|
-
super
|
811
|
+
super("dead")
|
812
812
|
end
|
813
813
|
|
814
814
|
# Add the given job to the Dead set.
|
@@ -1136,6 +1136,20 @@ module Sidekiq
|
|
1136
1136
|
end
|
1137
1137
|
end
|
1138
1138
|
end
|
1139
|
+
|
1140
|
+
##
|
1141
|
+
# Find the work which represents a job with the given JID.
|
1142
|
+
# *This is a slow O(n) operation*. Do not use for app logic.
|
1143
|
+
#
|
1144
|
+
# @param jid [String] the job identifier
|
1145
|
+
# @return [Sidekiq::Work] the work or nil
|
1146
|
+
def find_work_by_jid(jid)
|
1147
|
+
each do |_process_id, _thread_id, work|
|
1148
|
+
job = work.job
|
1149
|
+
return work if job.jid == jid
|
1150
|
+
end
|
1151
|
+
nil
|
1152
|
+
end
|
1139
1153
|
end
|
1140
1154
|
|
1141
1155
|
# Sidekiq::Work represents a job which is currently executing.
|
data/lib/sidekiq/cli.rb
CHANGED
@@ -38,7 +38,7 @@ module Sidekiq # :nodoc:
|
|
38
38
|
# Code within this method is not tested because it alters
|
39
39
|
# global process state irreversibly. PRs which improve the
|
40
40
|
# test coverage of Sidekiq::CLI are welcomed.
|
41
|
-
def run(boot_app: true)
|
41
|
+
def run(boot_app: true, warmup: true)
|
42
42
|
boot_application if boot_app
|
43
43
|
|
44
44
|
if environment == "development" && $stdout.tty? && @config.logger.formatter.is_a?(Sidekiq::Logger::Formatters::Pretty)
|
@@ -101,6 +101,8 @@ module Sidekiq # :nodoc:
|
|
101
101
|
# Touch middleware so it isn't lazy loaded by multiple threads, #3043
|
102
102
|
@config.server_middleware
|
103
103
|
|
104
|
+
::Process.warmup if warmup && ::Process.respond_to?(:warmup)
|
105
|
+
|
104
106
|
# Before this point, the process is initializing with just the main thread.
|
105
107
|
# Starting here the process will now have multiple threads running.
|
106
108
|
fire_event(:startup, reverse: false, reraise: true)
|
data/lib/sidekiq/deploy.rb
CHANGED
@@ -34,7 +34,7 @@ module Sidekiq
|
|
34
34
|
# handle an very common error in marking deploys:
|
35
35
|
# having every process mark its deploy, leading
|
36
36
|
# to N marks for each deploy. Instead we round the time
|
37
|
-
# to the minute so that
|
37
|
+
# to the minute so that multiple marks within that minute
|
38
38
|
# will all naturally rollup into one mark per minute.
|
39
39
|
whence = at.utc
|
40
40
|
floor = Time.utc(whence.year, whence.month, whence.mday, whence.hour, whence.min, 0)
|
data/lib/sidekiq/job.rb
CHANGED
@@ -109,7 +109,7 @@ module Sidekiq
|
|
109
109
|
m = "#{name}="
|
110
110
|
undef_method(m) if method_defined?(m) || private_method_defined?(m)
|
111
111
|
end
|
112
|
-
define_singleton_method("#{name}=") do |val|
|
112
|
+
define_singleton_method(:"#{name}=") do |val|
|
113
113
|
singleton_class.class_eval do
|
114
114
|
ACCESSOR_MUTEX.synchronize do
|
115
115
|
undef_method(synchronized_getter) if method_defined?(synchronized_getter) || private_method_defined?(synchronized_getter)
|
data/lib/sidekiq/launcher.rb
CHANGED
@@ -145,15 +145,17 @@ module Sidekiq
|
|
145
145
|
flush_stats
|
146
146
|
|
147
147
|
curstate = Processor::WORK_STATE.dup
|
148
|
+
curstate.transform_values! { |val| Sidekiq.dump_json(val) }
|
149
|
+
|
148
150
|
redis do |conn|
|
149
151
|
# work is the current set of executing jobs
|
150
152
|
work_key = "#{key}:work"
|
151
|
-
conn.
|
153
|
+
conn.multi do |transaction|
|
152
154
|
transaction.unlink(work_key)
|
153
|
-
curstate.
|
154
|
-
transaction.hset(work_key,
|
155
|
+
if curstate.size > 0
|
156
|
+
transaction.hset(work_key, curstate)
|
157
|
+
transaction.expire(work_key, 60)
|
155
158
|
end
|
156
|
-
transaction.expire(work_key, 60)
|
157
159
|
end
|
158
160
|
end
|
159
161
|
|
data/lib/sidekiq/logger.rb
CHANGED
data/lib/sidekiq/processor.rb
CHANGED
@@ -187,7 +187,7 @@ module Sidekiq
|
|
187
187
|
# we didn't properly finish it.
|
188
188
|
rescue Sidekiq::JobRetry::Handled => h
|
189
189
|
# this is the common case: job raised error and Sidekiq::JobRetry::Handled
|
190
|
-
# signals that we created a retry successfully. We can
|
190
|
+
# signals that we created a retry successfully. We can acknowledge the job.
|
191
191
|
ack = true
|
192
192
|
e = h.cause || h
|
193
193
|
handle_exception(e, {context: "Job raised exception", job: job_hash})
|
data/lib/sidekiq/rails.rb
CHANGED
data/lib/sidekiq/scheduled.rb
CHANGED
@@ -144,7 +144,7 @@ module Sidekiq
|
|
144
144
|
# In the example above, each process should schedule every 10 seconds on average. We special
|
145
145
|
# case smaller clusters to add 50% so they would sleep somewhere between 5 and 15 seconds.
|
146
146
|
# As we run more processes, the scheduling interval average will approach an even spread
|
147
|
-
# between 0 and poll interval so we don't need this
|
147
|
+
# between 0 and poll interval so we don't need this artificial boost.
|
148
148
|
#
|
149
149
|
count = process_count
|
150
150
|
interval = poll_interval_average(count)
|
data/lib/sidekiq/testing.rb
CHANGED
@@ -112,7 +112,7 @@ module Sidekiq
|
|
112
112
|
# The Queues class is only for testing the fake queue implementation.
|
113
113
|
# There are 2 data structures involved in tandem. This is due to the
|
114
114
|
# Rspec syntax of change(HardJob.jobs, :size). It keeps a reference
|
115
|
-
# to the array. Because the array was
|
115
|
+
# to the array. Because the array was derived from a filter of the total
|
116
116
|
# jobs enqueued, it appeared as though the array didn't change.
|
117
117
|
#
|
118
118
|
# To solve this, we'll keep 2 hashes containing the jobs. One with keys based
|
@@ -9,7 +9,14 @@ module Sidekiq
|
|
9
9
|
@redis_client = Client.new(pool: pool, config: config)
|
10
10
|
end
|
11
11
|
|
12
|
+
def batching?
|
13
|
+
Thread.current[:sidekiq_batch]
|
14
|
+
end
|
15
|
+
|
12
16
|
def push(item)
|
17
|
+
# 6160 we can't support both Sidekiq::Batch and transactions.
|
18
|
+
return @redis_client.push(item) if batching?
|
19
|
+
|
13
20
|
# pre-allocate the JID so we can return it immediately and
|
14
21
|
# save it to the database as part of the transaction.
|
15
22
|
item["jid"] ||= SecureRandom.hex(12)
|
data/lib/sidekiq/version.rb
CHANGED
data/lib/sidekiq/web/action.rb
CHANGED
@@ -22,6 +22,11 @@ module Sidekiq
|
|
22
22
|
throw :halt, [302, {Web::LOCATION => "#{request.base_url}#{location}"}, []]
|
23
23
|
end
|
24
24
|
|
25
|
+
def reload_page
|
26
|
+
current_location = request.referer.gsub(request.base_url, "")
|
27
|
+
redirect current_location
|
28
|
+
end
|
29
|
+
|
25
30
|
def params
|
26
31
|
indifferent_hash = Hash.new { |hash, key| hash[key.to_s] if Symbol === key }
|
27
32
|
|
@@ -49,9 +49,9 @@ module Sidekiq
|
|
49
49
|
|
50
50
|
head "/" do
|
51
51
|
# HEAD / is the cheapest heartbeat possible,
|
52
|
-
# it hits Redis to ensure connectivity
|
53
|
-
|
54
|
-
""
|
52
|
+
# it hits Redis to ensure connectivity and returns
|
53
|
+
# the size of the default queue
|
54
|
+
Sidekiq.redis { |c| c.llen("queue:default") }.to_s
|
55
55
|
end
|
56
56
|
|
57
57
|
get "/" do
|
@@ -394,6 +394,18 @@ module Sidekiq
|
|
394
394
|
erb :morgue
|
395
395
|
end
|
396
396
|
|
397
|
+
post "/change_locale" do
|
398
|
+
locale = params["locale"]
|
399
|
+
|
400
|
+
match = available_locales.find { |available|
|
401
|
+
locale == available
|
402
|
+
}
|
403
|
+
|
404
|
+
session[:locale] = match if match
|
405
|
+
|
406
|
+
reload_page
|
407
|
+
end
|
408
|
+
|
397
409
|
def call(env)
|
398
410
|
action = self.class.match(env)
|
399
411
|
return [404, {Rack::CONTENT_TYPE => "text/plain", Web::X_CASCADE => "pass"}, ["Not Found"]] unless action
|
@@ -56,7 +56,7 @@ module Sidekiq
|
|
56
56
|
end
|
57
57
|
|
58
58
|
def logger(env)
|
59
|
-
@logger ||=
|
59
|
+
@logger ||= env["rack.logger"] || ::Logger.new(env["rack.errors"])
|
60
60
|
end
|
61
61
|
|
62
62
|
def deny(env)
|
@@ -115,7 +115,7 @@ module Sidekiq
|
|
115
115
|
sess = session(env)
|
116
116
|
localtoken = sess[:csrf]
|
117
117
|
|
118
|
-
# Checks that Rack::Session::Cookie
|
118
|
+
# Checks that Rack::Session::Cookie actually contains the csrf token
|
119
119
|
return false if localtoken.nil?
|
120
120
|
|
121
121
|
# Rotate the session token after every use
|
data/lib/sidekiq/web/helpers.rb
CHANGED
@@ -121,6 +121,10 @@ module Sidekiq
|
|
121
121
|
#
|
122
122
|
# Inspiration taken from https://github.com/iain/http_accept_language/blob/master/lib/http_accept_language/parser.rb
|
123
123
|
def locale
|
124
|
+
# session[:locale] is set via the locale selector from the footer
|
125
|
+
# defined?(session) && session are used to avoid exceptions when running tests
|
126
|
+
return session[:locale] if defined?(session) && session&.[](:locale)
|
127
|
+
|
124
128
|
@locale ||= begin
|
125
129
|
matched_locale = user_preferred_languages.map { |preferred|
|
126
130
|
preferred_language = preferred.split("-", 2).first
|
@@ -340,7 +344,8 @@ module Sidekiq
|
|
340
344
|
end
|
341
345
|
|
342
346
|
def pollable?
|
343
|
-
|
347
|
+
# there's no point to refreshing the metrics pages every N seconds
|
348
|
+
!(current_path == "" || current_path.index("metrics"))
|
344
349
|
end
|
345
350
|
|
346
351
|
def retry_or_delete_or_kill(job, params)
|
@@ -47,6 +47,8 @@ function addListeners() {
|
|
47
47
|
scheduleLivePoll();
|
48
48
|
}
|
49
49
|
}
|
50
|
+
|
51
|
+
document.getElementById("locale-select").addEventListener("change", updateLocale);
|
50
52
|
}
|
51
53
|
|
52
54
|
function addPollingListeners(_event) {
|
@@ -175,3 +177,7 @@ function replacePage(text) {
|
|
175
177
|
function showError(error) {
|
176
178
|
console.error(error)
|
177
179
|
}
|
180
|
+
|
181
|
+
function updateLocale(event) {
|
182
|
+
event.target.form.submit();
|
183
|
+
};
|
@@ -731,3 +731,16 @@ div.interval-slider input {
|
|
731
731
|
canvas {
|
732
732
|
margin: 20px 0 30px;
|
733
733
|
}
|
734
|
+
|
735
|
+
#locale-select {
|
736
|
+
float: left;
|
737
|
+
margin: 8px 15px;
|
738
|
+
}
|
739
|
+
|
740
|
+
@media (max-width: 767px) {
|
741
|
+
#locale-select {
|
742
|
+
float: none;
|
743
|
+
width: auto;
|
744
|
+
margin: 15px auto;
|
745
|
+
}
|
746
|
+
}
|
data/web/views/_footer.erb
CHANGED
@@ -15,7 +15,19 @@
|
|
15
15
|
<p class="navbar-text"><a rel=help href="https://github.com/sidekiq/sidekiq/wiki">docs</a></p>
|
16
16
|
</li>
|
17
17
|
<li>
|
18
|
-
<
|
18
|
+
<form id="locale-form" class="form-inline" action="<%= root_path %>change_locale" method="post">
|
19
|
+
<%= csrf_tag %>
|
20
|
+
<label class="sr-only" for="locale">Language</label>
|
21
|
+
<select id="locale-select" class="form-control" name="locale">
|
22
|
+
<% available_locales.each do |locale_option| %>
|
23
|
+
<% if locale_option == locale %>
|
24
|
+
<option selected value="<%= locale_option %>"><%= locale_option %></option>
|
25
|
+
<% else %>
|
26
|
+
<option value="<%= locale_option %>"><%= locale_option %></option>
|
27
|
+
<% end %>
|
28
|
+
<% end %>
|
29
|
+
</select>
|
30
|
+
</form>
|
19
31
|
</li>
|
20
32
|
</ul>
|
21
33
|
</div>
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: sidekiq
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 7.2.
|
4
|
+
version: 7.2.3
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Mike Perham
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2024-
|
11
|
+
date: 2024-04-25 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: redis-client
|