sidekiq 7.0.9 → 7.1.1
Sign up to get free protection for your applications and to get access to all the features.
Potentially problematic release.
This version of sidekiq might be problematic. Click here for more details.
- checksums.yaml +4 -4
- data/Changes.md +30 -0
- data/bin/sidekiqload +21 -3
- data/lib/sidekiq/api.rb +32 -9
- data/lib/sidekiq/cli.rb +2 -1
- data/lib/sidekiq/client.rb +31 -20
- data/lib/sidekiq/component.rb +1 -1
- data/lib/sidekiq/config.rb +7 -3
- data/lib/sidekiq/job.rb +1 -5
- data/lib/sidekiq/job_retry.rb +4 -2
- data/lib/sidekiq/job_util.rb +2 -2
- data/lib/sidekiq/launcher.rb +1 -1
- data/lib/sidekiq/middleware/current_attributes.rb +55 -16
- data/lib/sidekiq/paginator.rb +1 -1
- data/lib/sidekiq/rails.rb +6 -0
- data/lib/sidekiq/redis_client_adapter.rb +5 -24
- data/lib/sidekiq/scheduled.rb +1 -1
- data/lib/sidekiq/version.rb +1 -1
- data/sidekiq.gemspec +1 -10
- data/web/locales/fr.yml +14 -0
- data/web/views/metrics_for_job.erb +1 -1
- metadata +5 -13
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: a5a020fb72399ae30db922c2c39fe63932c6a0f0ca2094c39d809df18b448331
|
4
|
+
data.tar.gz: 50e6dfc4ab7db79279463fc70cc52226e7fe1a58188b6a0f3f2a774b76428a76
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 169a24acd3011faf7d449f7fbf08d2e62125f0802791afdac7db992950ffdf53e768bc6549ccdd46e1868b07c5b367cb574be613704b7da658927d21b75bb320
|
7
|
+
data.tar.gz: 165d6d1fb808f53731ff460c992bc5103bb0f66e619a9aed22561aee9c21e09fc59d8f1f388e52acde68eefdf81413953bd1dda6bf32ded0dbd9980a247a12e4
|
data/Changes.md
CHANGED
@@ -2,6 +2,31 @@
|
|
2
2
|
|
3
3
|
[Sidekiq Changes](https://github.com/sidekiq/sidekiq/blob/main/Changes.md) | [Sidekiq Pro Changes](https://github.com/sidekiq/sidekiq/blob/main/Pro-Changes.md) | [Sidekiq Enterprise Changes](https://github.com/sidekiq/sidekiq/blob/main/Ent-Changes.md)
|
4
4
|
|
5
|
+
7.1.1
|
6
|
+
----------
|
7
|
+
|
8
|
+
- Support multiple CurrentAttributes [#5904]
|
9
|
+
- Speed up latency fetch with large queues on Redis <7 [#5910]
|
10
|
+
- Allow a larger default client pool [#5886]
|
11
|
+
- Ensure Sidekiq.options[:environment] == RAILS_ENV [#5932]
|
12
|
+
|
13
|
+
7.1.0
|
14
|
+
----------
|
15
|
+
|
16
|
+
- Improve display of ActiveJob arguments in Web UI [#5825, cover]
|
17
|
+
- Update `push_bulk` to push `batch_size` jobs at a time and allow laziness [#5827, fatkodima]
|
18
|
+
This allows Sidekiq::Client to push unlimited jobs as long as it has enough memory for the batch_size.
|
19
|
+
- Update `perform_bulk` to use `push_bulk` internally.
|
20
|
+
- Change return value of `push_bulk` to map 1-to-1 with arguments.
|
21
|
+
If you call `push_bulk(args: [[1], [2], [3]])`, you will now always get
|
22
|
+
an array of 3 values as the result: `["jid1", nil, "jid3"]` where nil means
|
23
|
+
that particular job did not push successfully (possibly due to middleware
|
24
|
+
stopping it). Previously nil values were removed so it was impossible to tell
|
25
|
+
which jobs pushed successfully and which did not.
|
26
|
+
- Migrate away from all deprecated Redis commands [#5788]
|
27
|
+
Sidekiq will now print a warning if you use one of those deprecated commands.
|
28
|
+
- Prefix all Sidekiq thread names [#5872]
|
29
|
+
|
5
30
|
7.0.9
|
6
31
|
----------
|
7
32
|
|
@@ -84,6 +109,11 @@ end
|
|
84
109
|
- Job Execution metrics!!!
|
85
110
|
- See `docs/7.0-Upgrade.md` for release notes
|
86
111
|
|
112
|
+
6.5.9
|
113
|
+
----------
|
114
|
+
|
115
|
+
- Ensure Sidekiq.options[:environment] == RAILS_ENV [#5932]
|
116
|
+
|
87
117
|
6.5.8
|
88
118
|
----------
|
89
119
|
|
data/bin/sidekiqload
CHANGED
@@ -1,5 +1,23 @@
|
|
1
1
|
#!/usr/bin/env ruby
|
2
2
|
|
3
|
+
#
|
4
|
+
# bin/sidekiqload is a helpful script to load test and
|
5
|
+
# performance tune Sidekiq's core. It creates 500,000 no-op
|
6
|
+
# jobs and executes them as fast as possible.
|
7
|
+
# Example Usage:
|
8
|
+
#
|
9
|
+
# > RUBY_YJIT_ENABLE=1 LATENCY=0 THREADS=10 bin/sidekiqload
|
10
|
+
# Result: Done, 500000 jobs in 20.264945 sec, 24673 jobs/sec
|
11
|
+
#
|
12
|
+
# Use LATENCY=1 to get a more real world network setup
|
13
|
+
# but you'll need to setup and start toxiproxy as noted below.
|
14
|
+
#
|
15
|
+
# Use AJ=1 to test ActiveJob instead of plain old Sidekiq::Jobs so
|
16
|
+
# you can see the runtime performance difference between the two APIs.
|
17
|
+
#
|
18
|
+
# None of this script is considered a public API and may change over time.
|
19
|
+
#
|
20
|
+
|
3
21
|
# Quiet some warnings we see when running in warning mode:
|
4
22
|
# RUBYOPT=-w bundle exec sidekiq
|
5
23
|
$TESTING = false
|
@@ -32,7 +50,7 @@ if ENV["AJ"]
|
|
32
50
|
ActiveJob::Base.logger.level = Logger::WARN
|
33
51
|
|
34
52
|
class LoadJob < ActiveJob::Base
|
35
|
-
def perform(idx, ts=nil)
|
53
|
+
def perform(idx, ts = nil)
|
36
54
|
puts(Time.now.to_f - ts) if !ts.nil?
|
37
55
|
end
|
38
56
|
end
|
@@ -219,11 +237,11 @@ end
|
|
219
237
|
ll = Loader.new
|
220
238
|
ll.configure
|
221
239
|
|
222
|
-
|
240
|
+
if ENV["WARM"]
|
223
241
|
ll.setup
|
224
242
|
ll.run("warmup")
|
225
243
|
end
|
226
244
|
|
227
245
|
ll.setup
|
228
|
-
ll.run("
|
246
|
+
ll.run("load")
|
229
247
|
ll.done
|
data/lib/sidekiq/api.rb
CHANGED
@@ -92,11 +92,11 @@ module Sidekiq
|
|
92
92
|
pipeline.zcard("retry")
|
93
93
|
pipeline.zcard("dead")
|
94
94
|
pipeline.scard("processes")
|
95
|
-
pipeline.
|
95
|
+
pipeline.lindex("queue:default", -1)
|
96
96
|
end
|
97
97
|
}
|
98
98
|
|
99
|
-
default_queue_latency = if (entry = pipe1_res[6]
|
99
|
+
default_queue_latency = if (entry = pipe1_res[6])
|
100
100
|
job = begin
|
101
101
|
Sidekiq.load_json(entry)
|
102
102
|
rescue
|
@@ -264,8 +264,8 @@ module Sidekiq
|
|
264
264
|
# @return [Float] in seconds
|
265
265
|
def latency
|
266
266
|
entry = Sidekiq.redis { |conn|
|
267
|
-
conn.
|
268
|
-
}
|
267
|
+
conn.lindex(@rname, -1)
|
268
|
+
}
|
269
269
|
return 0 unless entry
|
270
270
|
job = Sidekiq.load_json(entry)
|
271
271
|
now = Time.now.to_f
|
@@ -391,13 +391,13 @@ module Sidekiq
|
|
391
391
|
def display_args
|
392
392
|
# Unwrap known wrappers so they show up in a human-friendly manner in the Web UI
|
393
393
|
@display_args ||= if klass == "ActiveJob::QueueAdapters::SidekiqAdapter::JobWrapper"
|
394
|
-
job_args = self["wrapped"] ? args[0]["arguments"] : []
|
394
|
+
job_args = self["wrapped"] ? deserialize_argument(args[0]["arguments"]) : []
|
395
395
|
if (self["wrapped"] || args[0]) == "ActionMailer::DeliveryJob"
|
396
396
|
# remove MailerClass, mailer_method and 'deliver_now'
|
397
397
|
job_args.drop(3)
|
398
398
|
elsif (self["wrapped"] || args[0]) == "ActionMailer::MailDeliveryJob"
|
399
399
|
# remove MailerClass, mailer_method and 'deliver_now'
|
400
|
-
job_args.drop(3).first
|
400
|
+
job_args.drop(3).first.values_at("params", "args")
|
401
401
|
else
|
402
402
|
job_args
|
403
403
|
end
|
@@ -467,6 +467,29 @@ module Sidekiq
|
|
467
467
|
|
468
468
|
private
|
469
469
|
|
470
|
+
ACTIVE_JOB_PREFIX = "_aj_"
|
471
|
+
GLOBALID_KEY = "_aj_globalid"
|
472
|
+
|
473
|
+
def deserialize_argument(argument)
|
474
|
+
case argument
|
475
|
+
when Array
|
476
|
+
argument.map { |arg| deserialize_argument(arg) }
|
477
|
+
when Hash
|
478
|
+
if serialized_global_id?(argument)
|
479
|
+
argument[GLOBALID_KEY]
|
480
|
+
else
|
481
|
+
argument.transform_values { |v| deserialize_argument(v) }
|
482
|
+
.reject { |k, _| k.start_with?(ACTIVE_JOB_PREFIX) }
|
483
|
+
end
|
484
|
+
else
|
485
|
+
argument
|
486
|
+
end
|
487
|
+
end
|
488
|
+
|
489
|
+
def serialized_global_id?(hash)
|
490
|
+
hash.size == 1 && hash.include?(GLOBALID_KEY)
|
491
|
+
end
|
492
|
+
|
470
493
|
def uncompress_backtrace(backtrace)
|
471
494
|
decoded = Base64.decode64(backtrace)
|
472
495
|
uncompressed = Zlib::Inflate.inflate(decoded)
|
@@ -548,7 +571,7 @@ module Sidekiq
|
|
548
571
|
def remove_job
|
549
572
|
Sidekiq.redis do |conn|
|
550
573
|
results = conn.multi { |transaction|
|
551
|
-
transaction.
|
574
|
+
transaction.zrange(parent.name, score, score, "BYSCORE")
|
552
575
|
transaction.zremrangebyscore(parent.name, score, score)
|
553
576
|
}.first
|
554
577
|
|
@@ -683,7 +706,7 @@ module Sidekiq
|
|
683
706
|
end
|
684
707
|
|
685
708
|
elements = Sidekiq.redis { |conn|
|
686
|
-
conn.
|
709
|
+
conn.zrange(name, begin_score, end_score, "BYSCORE", withscores: true)
|
687
710
|
}
|
688
711
|
|
689
712
|
elements.each_with_object([]) do |element, result|
|
@@ -724,7 +747,7 @@ module Sidekiq
|
|
724
747
|
# @api private
|
725
748
|
def delete_by_jid(score, jid)
|
726
749
|
Sidekiq.redis do |conn|
|
727
|
-
elements = conn.
|
750
|
+
elements = conn.zrange(name, score, score, "BYSCORE")
|
728
751
|
elements.each do |element|
|
729
752
|
if element.index(jid)
|
730
753
|
message = Sidekiq.load_json(element)
|
data/lib/sidekiq/cli.rb
CHANGED
@@ -230,6 +230,7 @@ module Sidekiq # :nodoc:
|
|
230
230
|
# Both Sinatra 2.0+ and Sidekiq support this term.
|
231
231
|
# RAILS_ENV and RACK_ENV are there for legacy support.
|
232
232
|
@environment = cli_env || ENV["APP_ENV"] || ENV["RAILS_ENV"] || ENV["RACK_ENV"] || "development"
|
233
|
+
config[:environment] = @environment
|
233
234
|
end
|
234
235
|
|
235
236
|
def symbolize_keys_deep!(hash)
|
@@ -396,7 +397,7 @@ module Sidekiq # :nodoc:
|
|
396
397
|
end
|
397
398
|
|
398
399
|
def parse_config(path)
|
399
|
-
erb = ERB.new(File.read(path))
|
400
|
+
erb = ERB.new(File.read(path), trim_mode: "-")
|
400
401
|
erb.filename = File.expand_path(path)
|
401
402
|
opts = YAML.safe_load(erb.result, permitted_classes: [Symbol], aliases: true) || {}
|
402
403
|
|
data/lib/sidekiq/client.rb
CHANGED
@@ -96,8 +96,9 @@ module Sidekiq
|
|
96
96
|
|
97
97
|
##
|
98
98
|
# Push a large number of jobs to Redis. This method cuts out the redis
|
99
|
-
# network round trip latency.
|
100
|
-
# 1000
|
99
|
+
# network round trip latency. It pushes jobs in batches if more than
|
100
|
+
# `:batch_size` (1000 by default) of jobs are passed. I wouldn't recommend making `:batch_size`
|
101
|
+
# larger than 1000 but YMMV based on network quality, size of job args, etc.
|
101
102
|
# A large number of jobs can cause a bit of Redis command processing latency.
|
102
103
|
#
|
103
104
|
# Takes the same arguments as #push except that args is expected to be
|
@@ -105,13 +106,15 @@ module Sidekiq
|
|
105
106
|
# is run through the client middleware pipeline and each job gets its own Job ID
|
106
107
|
# as normal.
|
107
108
|
#
|
108
|
-
# Returns an array of the of pushed jobs' jids
|
109
|
-
#
|
109
|
+
# Returns an array of the of pushed jobs' jids, may contain nils if any client middleware
|
110
|
+
# prevented a job push.
|
111
|
+
#
|
112
|
+
# Example (pushing jobs in batches):
|
113
|
+
# push_bulk('class' => 'MyJob', 'args' => (1..100_000).to_a, batch_size: 1_000)
|
114
|
+
#
|
110
115
|
def push_bulk(items)
|
116
|
+
batch_size = items.delete(:batch_size) || items.delete("batch_size") || 1_000
|
111
117
|
args = items["args"]
|
112
|
-
raise ArgumentError, "Bulk arguments must be an Array of Arrays: [[1], [2]]" unless args.is_a?(Array) && args.all?(Array)
|
113
|
-
return [] if args.empty? # no jobs to push
|
114
|
-
|
115
118
|
at = items.delete("at")
|
116
119
|
raise ArgumentError, "Job 'at' must be a Numeric or an Array of Numeric timestamps" if at && (Array(at).empty? || !Array(at).all? { |entry| entry.is_a?(Numeric) })
|
117
120
|
raise ArgumentError, "Job 'at' Array must have same size as 'args' Array" if at.is_a?(Array) && at.size != args.size
|
@@ -120,18 +123,26 @@ module Sidekiq
|
|
120
123
|
raise ArgumentError, "Explicitly passing 'jid' when pushing more than one job is not supported" if jid && args.size > 1
|
121
124
|
|
122
125
|
normed = normalize_item(items)
|
123
|
-
|
124
|
-
|
125
|
-
|
126
|
-
|
127
|
-
|
128
|
-
copy
|
129
|
-
|
130
|
-
|
131
|
-
|
126
|
+
result = args.each_slice(batch_size).flat_map do |slice|
|
127
|
+
raise ArgumentError, "Bulk arguments must be an Array of Arrays: [[1], [2]]" unless slice.is_a?(Array) && slice.all?(Array)
|
128
|
+
break [] if slice.empty? # no jobs to push
|
129
|
+
|
130
|
+
payloads = slice.map.with_index { |job_args, index|
|
131
|
+
copy = normed.merge("args" => job_args, "jid" => SecureRandom.hex(12))
|
132
|
+
copy["at"] = (at.is_a?(Array) ? at[index] : at) if at
|
133
|
+
result = middleware.invoke(items["class"], copy, copy["queue"], @redis_pool) do
|
134
|
+
verify_json(copy)
|
135
|
+
copy
|
136
|
+
end
|
137
|
+
result || nil
|
138
|
+
}
|
139
|
+
|
140
|
+
to_push = payloads.compact
|
141
|
+
raw_push(to_push) unless to_push.empty?
|
142
|
+
payloads.map { |payload| payload&.[]("jid") }
|
143
|
+
end
|
132
144
|
|
133
|
-
|
134
|
-
payloads.collect { |payload| payload["jid"] }
|
145
|
+
result.is_a?(Enumerator::Lazy) ? result.force : result
|
135
146
|
end
|
136
147
|
|
137
148
|
# Allows sharding of jobs across any number of Redis instances. All jobs
|
@@ -160,8 +171,8 @@ module Sidekiq
|
|
160
171
|
new.push(item)
|
161
172
|
end
|
162
173
|
|
163
|
-
def push_bulk(
|
164
|
-
new.push_bulk(
|
174
|
+
def push_bulk(...)
|
175
|
+
new.push_bulk(...)
|
165
176
|
end
|
166
177
|
|
167
178
|
# Resque compatibility helpers. Note all helpers
|
data/lib/sidekiq/component.rb
CHANGED
data/lib/sidekiq/config.rb
CHANGED
@@ -30,7 +30,8 @@ module Sidekiq
|
|
30
30
|
},
|
31
31
|
dead_max_jobs: 10_000,
|
32
32
|
dead_timeout_in_seconds: 180 * 24 * 60 * 60, # 6 months
|
33
|
-
reloader: proc { |&block| block.call }
|
33
|
+
reloader: proc { |&block| block.call },
|
34
|
+
backtrace_cleaner: ->(backtrace) { backtrace }
|
34
35
|
}
|
35
36
|
|
36
37
|
ERROR_HANDLER = ->(ex, ctx) {
|
@@ -38,7 +39,10 @@ module Sidekiq
|
|
38
39
|
l = cfg.logger
|
39
40
|
l.warn(Sidekiq.dump_json(ctx)) unless ctx.empty?
|
40
41
|
l.warn("#{ex.class.name}: #{ex.message}")
|
41
|
-
|
42
|
+
unless ex.backtrace.nil?
|
43
|
+
backtrace = cfg[:backtrace_cleaner].call(ex.backtrace)
|
44
|
+
l.warn(backtrace.join("\n"))
|
45
|
+
end
|
42
46
|
}
|
43
47
|
|
44
48
|
def initialize(options = {})
|
@@ -123,7 +127,7 @@ module Sidekiq
|
|
123
127
|
private def local_redis_pool
|
124
128
|
# this is our internal client/housekeeping pool. each capsule has its
|
125
129
|
# own pool for executing threads.
|
126
|
-
@redis ||= new_redis_pool(
|
130
|
+
@redis ||= new_redis_pool(10, "internal")
|
127
131
|
end
|
128
132
|
|
129
133
|
def new_redis_pool(size, name = "unset")
|
data/lib/sidekiq/job.rb
CHANGED
@@ -239,11 +239,7 @@ module Sidekiq
|
|
239
239
|
|
240
240
|
def perform_bulk(args, batch_size: 1_000)
|
241
241
|
client = @klass.build_client
|
242
|
-
|
243
|
-
client.push_bulk(@opts.merge("class" => @klass, "args" => slice))
|
244
|
-
end
|
245
|
-
|
246
|
-
result.is_a?(Enumerator::Lazy) ? result.force : result
|
242
|
+
client.push_bulk(@opts.merge("class" => @klass, "args" => args, :batch_size => batch_size))
|
247
243
|
end
|
248
244
|
|
249
245
|
# +interval+ must be a timestamp, numeric or something that acts
|
data/lib/sidekiq/job_retry.rb
CHANGED
@@ -71,6 +71,7 @@ module Sidekiq
|
|
71
71
|
def initialize(capsule)
|
72
72
|
@config = @capsule = capsule
|
73
73
|
@max_retries = Sidekiq.default_configuration[:max_retries] || DEFAULT_MAX_RETRY_ATTEMPTS
|
74
|
+
@backtrace_cleaner = Sidekiq.default_configuration[:backtrace_cleaner]
|
74
75
|
end
|
75
76
|
|
76
77
|
# The global retry handler requires only the barest of data.
|
@@ -159,10 +160,11 @@ module Sidekiq
|
|
159
160
|
end
|
160
161
|
|
161
162
|
if msg["backtrace"]
|
163
|
+
backtrace = @backtrace_cleaner.call(exception.backtrace)
|
162
164
|
lines = if msg["backtrace"] == true
|
163
|
-
|
165
|
+
backtrace
|
164
166
|
else
|
165
|
-
|
167
|
+
backtrace[0...msg["backtrace"].to_i]
|
166
168
|
end
|
167
169
|
|
168
170
|
msg["error_backtrace"] = compress_backtrace(lines)
|
data/lib/sidekiq/job_util.rb
CHANGED
@@ -9,7 +9,7 @@ module Sidekiq
|
|
9
9
|
|
10
10
|
def validate(item)
|
11
11
|
raise(ArgumentError, "Job must be a Hash with 'class' and 'args' keys: `#{item}`") unless item.is_a?(Hash) && item.key?("class") && item.key?("args")
|
12
|
-
raise(ArgumentError, "Job args must be an Array: `#{item}`") unless item["args"].is_a?(Array)
|
12
|
+
raise(ArgumentError, "Job args must be an Array: `#{item}`") unless item["args"].is_a?(Array) || item["args"].is_a?(Enumerator::Lazy)
|
13
13
|
raise(ArgumentError, "Job class must be either a Class or String representation of the class name: `#{item}`") unless item["class"].is_a?(Class) || item["class"].is_a?(String)
|
14
14
|
raise(ArgumentError, "Job 'at' must be a Numeric timestamp: `#{item}`") if item.key?("at") && !item["at"].is_a?(Numeric)
|
15
15
|
raise(ArgumentError, "Job tags must be an Array: `#{item}`") if item["tags"] && !item["tags"].is_a?(Array)
|
@@ -24,7 +24,7 @@ module Sidekiq
|
|
24
24
|
if (unsafe_item = json_unsafe?(args))
|
25
25
|
msg = <<~EOM
|
26
26
|
Job arguments to #{job_class} must be native JSON types, but #{unsafe_item.inspect} is a #{unsafe_item.class}.
|
27
|
-
See https://github.com/sidekiq/sidekiq/wiki/Best-Practices
|
27
|
+
See https://github.com/sidekiq/sidekiq/wiki/Best-Practices
|
28
28
|
To disable this error, add `Sidekiq.strict_args!(false)` to your initializer.
|
29
29
|
EOM
|
30
30
|
|
data/lib/sidekiq/launcher.rb
CHANGED
@@ -166,7 +166,7 @@ module Sidekiq
|
|
166
166
|
conn.multi { |transaction|
|
167
167
|
transaction.sadd("processes", [key])
|
168
168
|
transaction.exists(key)
|
169
|
-
transaction.
|
169
|
+
transaction.hset(key, "info", to_json,
|
170
170
|
"busy", curstate.size,
|
171
171
|
"beat", Time.now.to_f,
|
172
172
|
"rtt_us", rtt,
|
@@ -7,26 +7,32 @@ module Sidekiq
|
|
7
7
|
# This can be useful for multi-tenancy, i18n locale, timezone, any implicit
|
8
8
|
# per-request attribute. See +ActiveSupport::CurrentAttributes+.
|
9
9
|
#
|
10
|
+
# For multiple current attributes, pass an array of current attributes.
|
11
|
+
#
|
10
12
|
# @example
|
11
13
|
#
|
12
14
|
# # in your initializer
|
13
15
|
# require "sidekiq/middleware/current_attributes"
|
14
16
|
# Sidekiq::CurrentAttributes.persist("Myapp::Current")
|
17
|
+
# # or multiple current attributes
|
18
|
+
# Sidekiq::CurrentAttributes.persist(["Myapp::Current", "Myapp::OtherCurrent"])
|
15
19
|
#
|
16
20
|
module CurrentAttributes
|
17
21
|
class Save
|
18
22
|
include Sidekiq::ClientMiddleware
|
19
23
|
|
20
|
-
def initialize(
|
21
|
-
@
|
24
|
+
def initialize(cattrs)
|
25
|
+
@cattrs = cattrs
|
22
26
|
end
|
23
27
|
|
24
28
|
def call(_, job, _, _)
|
25
|
-
|
26
|
-
|
27
|
-
|
28
|
-
|
29
|
-
|
29
|
+
@cattrs.each do |(key, strklass)|
|
30
|
+
if !job.has_key?(key)
|
31
|
+
attrs = strklass.constantize.attributes
|
32
|
+
# Retries can push the job N times, we don't
|
33
|
+
# want retries to reset cattr. #5692, #5090
|
34
|
+
job[key] = attrs if attrs.any?
|
35
|
+
end
|
30
36
|
end
|
31
37
|
yield
|
32
38
|
end
|
@@ -35,22 +41,55 @@ module Sidekiq
|
|
35
41
|
class Load
|
36
42
|
include Sidekiq::ServerMiddleware
|
37
43
|
|
38
|
-
def initialize(
|
39
|
-
@
|
44
|
+
def initialize(cattrs)
|
45
|
+
@cattrs = cattrs
|
40
46
|
end
|
41
47
|
|
42
48
|
def call(_, job, _, &block)
|
43
|
-
|
44
|
-
|
45
|
-
|
46
|
-
|
49
|
+
cattrs_to_reset = []
|
50
|
+
|
51
|
+
@cattrs.each do |(key, strklass)|
|
52
|
+
if job.has_key?(key)
|
53
|
+
constklass = strklass.constantize
|
54
|
+
cattrs_to_reset << constklass
|
55
|
+
|
56
|
+
job[key].each do |(attribute, value)|
|
57
|
+
constklass.public_send("#{attribute}=", value)
|
58
|
+
end
|
59
|
+
end
|
47
60
|
end
|
61
|
+
|
62
|
+
yield
|
63
|
+
ensure
|
64
|
+
cattrs_to_reset.each(&:reset)
|
48
65
|
end
|
49
66
|
end
|
50
67
|
|
51
|
-
|
52
|
-
|
53
|
-
|
68
|
+
class << self
|
69
|
+
def persist(klass_or_array, config = Sidekiq.default_configuration)
|
70
|
+
cattrs = build_cattrs_hash(klass_or_array)
|
71
|
+
|
72
|
+
config.client_middleware.add Save, cattrs
|
73
|
+
config.server_middleware.add Load, cattrs
|
74
|
+
end
|
75
|
+
|
76
|
+
private
|
77
|
+
|
78
|
+
def build_cattrs_hash(klass_or_array)
|
79
|
+
if klass_or_array.is_a?(Array)
|
80
|
+
{}.tap do |hash|
|
81
|
+
klass_or_array.each_with_index do |klass, index|
|
82
|
+
hash[key_at(index)] = klass.to_s
|
83
|
+
end
|
84
|
+
end
|
85
|
+
else
|
86
|
+
{key_at(0) => klass_or_array.to_s}
|
87
|
+
end
|
88
|
+
end
|
89
|
+
|
90
|
+
def key_at(index)
|
91
|
+
(index == 0) ? "cattr" : "cattr_#{index}"
|
92
|
+
end
|
54
93
|
end
|
55
94
|
end
|
56
95
|
end
|
data/lib/sidekiq/paginator.rb
CHANGED
@@ -19,7 +19,7 @@ module Sidekiq
|
|
19
19
|
total_size, items = conn.multi { |transaction|
|
20
20
|
transaction.zcard(key)
|
21
21
|
if rev
|
22
|
-
transaction.
|
22
|
+
transaction.zrange(key, starting, ending, "REV", withscores: true)
|
23
23
|
else
|
24
24
|
transaction.zrange(key, starting, ending, withscores: true)
|
25
25
|
end
|
data/lib/sidekiq/rails.rb
CHANGED
@@ -50,6 +50,12 @@ module Sidekiq
|
|
50
50
|
end
|
51
51
|
end
|
52
52
|
|
53
|
+
initializer "sidekiq.backtrace_cleaner" do
|
54
|
+
Sidekiq.configure_server do |config|
|
55
|
+
config[:backtrace_cleaner] = ->(backtrace) { ::Rails.backtrace_cleaner.clean(backtrace) }
|
56
|
+
end
|
57
|
+
end
|
58
|
+
|
53
59
|
# This hook happens after all initializers are run, just before returning
|
54
60
|
# from config/environment.rb back to sidekiq/cli.rb.
|
55
61
|
#
|
@@ -1,5 +1,6 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
+
require "set"
|
3
4
|
require "redis_client"
|
4
5
|
require "redis_client/decorator"
|
5
6
|
|
@@ -8,13 +9,14 @@ module Sidekiq
|
|
8
9
|
BaseError = RedisClient::Error
|
9
10
|
CommandError = RedisClient::CommandError
|
10
11
|
|
12
|
+
# You can add/remove items or clear the whole thing if you don't want deprecation warnings.
|
13
|
+
DEPRECATED_COMMANDS = %i[rpoplpush zrangebyscore zrevrange zrevrangebyscore getset hmset setex setnx].to_set
|
14
|
+
|
11
15
|
module CompatMethods
|
12
|
-
# TODO Deprecate and remove this
|
13
16
|
def info
|
14
17
|
@client.call("INFO") { |i| i.lines(chomp: true).map { |l| l.split(":", 2) }.select { |l| l.size == 2 }.to_h }
|
15
18
|
end
|
16
19
|
|
17
|
-
# TODO Deprecate and remove this
|
18
20
|
def evalsha(sha, keys, argv)
|
19
21
|
@client.call("EVALSHA", sha, keys.size, *keys, *argv)
|
20
22
|
end
|
@@ -24,6 +26,7 @@ module Sidekiq
|
|
24
26
|
# this allows us to use methods like `conn.hmset(...)` instead of having to use
|
25
27
|
# redis-client's native `conn.call("hmset", ...)`
|
26
28
|
def method_missing(*args, &block)
|
29
|
+
warn("[sidekiq#5788] Redis has deprecated the `#{args.first}`command, called at #{caller(1..1)}") if DEPRECATED_COMMANDS.include?(args.first)
|
27
30
|
@client.call(*args, *block)
|
28
31
|
end
|
29
32
|
ruby2_keywords :method_missing if respond_to?(:ruby2_keywords, true)
|
@@ -39,28 +42,6 @@ module Sidekiq
|
|
39
42
|
def config
|
40
43
|
@client.config
|
41
44
|
end
|
42
|
-
|
43
|
-
def message
|
44
|
-
yield nil, @queue.pop
|
45
|
-
end
|
46
|
-
|
47
|
-
# NB: this method does not return
|
48
|
-
def subscribe(chan)
|
49
|
-
@queue = ::Queue.new
|
50
|
-
|
51
|
-
pubsub = @client.pubsub
|
52
|
-
pubsub.call("subscribe", chan)
|
53
|
-
|
54
|
-
loop do
|
55
|
-
evt = pubsub.next_event
|
56
|
-
next if evt.nil?
|
57
|
-
next unless evt[0] == "message" && evt[1] == chan
|
58
|
-
|
59
|
-
(_, _, msg) = evt
|
60
|
-
@queue << msg
|
61
|
-
yield self
|
62
|
-
end
|
63
|
-
end
|
64
45
|
end
|
65
46
|
|
66
47
|
def initialize(options)
|
data/lib/sidekiq/scheduled.rb
CHANGED
@@ -12,7 +12,7 @@ module Sidekiq
|
|
12
12
|
|
13
13
|
LUA_ZPOPBYSCORE = <<~LUA
|
14
14
|
local key, now = KEYS[1], ARGV[1]
|
15
|
-
local jobs = redis.call("
|
15
|
+
local jobs = redis.call("zrange", key, "-inf", now, "byscore", "limit", 0, 1)
|
16
16
|
if jobs[1] then
|
17
17
|
redis.call("zrem", key, jobs[1])
|
18
18
|
return jobs[1]
|
data/lib/sidekiq/version.rb
CHANGED
data/sidekiq.gemspec
CHANGED
@@ -23,17 +23,8 @@ Gem::Specification.new do |gem|
|
|
23
23
|
"rubygems_mfa_required" => "true"
|
24
24
|
}
|
25
25
|
|
26
|
-
gem.add_dependency "redis-client", ">= 0.
|
26
|
+
gem.add_dependency "redis-client", ">= 0.14.0"
|
27
27
|
gem.add_dependency "connection_pool", ">= 2.3.0"
|
28
28
|
gem.add_dependency "rack", ">= 2.2.4"
|
29
29
|
gem.add_dependency "concurrent-ruby", "< 2"
|
30
|
-
gem.post_install_message = <<~EOM
|
31
|
-
|
32
|
-
Welcome to Sidekiq 7.0!
|
33
|
-
|
34
|
-
1. Use `gem 'sidekiq', '<7'` in your Gemfile if you don't want this new version.
|
35
|
-
2. Read the release notes at https://github.com/sidekiq/sidekiq/blob/main/docs/7.0-Upgrade.md
|
36
|
-
3. If you have problems, search for open/closed issues at https://github.com/sidekiq/sidekiq/issues/
|
37
|
-
|
38
|
-
EOM
|
39
30
|
end
|
data/web/locales/fr.yml
CHANGED
@@ -17,14 +17,17 @@ fr:
|
|
17
17
|
DeadJobs: Tâches mortes
|
18
18
|
Delete: Supprimer
|
19
19
|
DeleteAll: Tout supprimer
|
20
|
+
Deploy: Déploiement
|
20
21
|
Enqueued: En attente
|
21
22
|
Error: Erreur
|
22
23
|
ErrorBacktrace: Backtrace d’erreur
|
23
24
|
ErrorClass: Classe d’erreur
|
24
25
|
ErrorMessage: Message d’erreur
|
26
|
+
ExecutionTime: Temps d'exécution
|
25
27
|
Extras: Extras
|
26
28
|
Failed: Échouées
|
27
29
|
Failures: Echecs
|
30
|
+
Failure: Echec
|
28
31
|
GoBack: ← Retour
|
29
32
|
History: Historique
|
30
33
|
Job: Tâche
|
@@ -35,6 +38,7 @@ fr:
|
|
35
38
|
Latency: Latence
|
36
39
|
LivePoll: Temps réel
|
37
40
|
MemoryUsage: Mémoire utilisée
|
41
|
+
Name: Nom
|
38
42
|
Namespace: Namespace
|
39
43
|
NextRetry: Prochain essai
|
40
44
|
NoDeadJobsFound: Aucune tâche morte n'a été trouvée
|
@@ -63,6 +67,7 @@ fr:
|
|
63
67
|
RetryNow: Réessayer maintenant
|
64
68
|
Scheduled: Planifiées
|
65
69
|
ScheduledJobs: Tâches planifiées
|
70
|
+
Seconds: Secondes
|
66
71
|
ShowAll: Tout montrer
|
67
72
|
SixMonths: 6 mois
|
68
73
|
Size: Taille
|
@@ -71,6 +76,8 @@ fr:
|
|
71
76
|
Stop: Arrêter
|
72
77
|
StopAll: Tout arrêter
|
73
78
|
StopPolling: Arrêt du temps réel
|
79
|
+
Success: Succès
|
80
|
+
Summary: Résumé
|
74
81
|
Thread: Thread
|
75
82
|
Threads: Threads
|
76
83
|
ThreeMonths: 3 mois
|
@@ -83,3 +90,10 @@ fr:
|
|
83
90
|
Worker: Travailleur
|
84
91
|
active: actif
|
85
92
|
idle: inactif
|
93
|
+
Metrics: Métriques
|
94
|
+
NoDataFound: Aucune donnée disponible
|
95
|
+
TotalExecutionTime: Temps d'exécution total
|
96
|
+
AvgExecutionTime: Temps d'exécution moyen
|
97
|
+
Context: Contexte
|
98
|
+
Bucket: Bucket
|
99
|
+
NoJobMetricsFound: Aucune statistique de tâche récente n'a été trouvée
|
@@ -5,7 +5,7 @@
|
|
5
5
|
|
6
6
|
<%
|
7
7
|
job_result = @query_result.job_results[@name]
|
8
|
-
hist_totals = job_result.hist.values.first.zip(*job_result.hist.values[1..-1]).map(&:sum)
|
8
|
+
hist_totals = job_result.hist.values.first.zip(*job_result.hist.values[1..-1]).map(&:sum).reverse
|
9
9
|
bucket_labels = Sidekiq::Metrics::Histogram::LABELS
|
10
10
|
bucket_intervals = Sidekiq::Metrics::Histogram::BUCKET_INTERVALS
|
11
11
|
%>
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: sidekiq
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 7.
|
4
|
+
version: 7.1.1
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Mike Perham
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2023-
|
11
|
+
date: 2023-05-25 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: redis-client
|
@@ -16,14 +16,14 @@ dependencies:
|
|
16
16
|
requirements:
|
17
17
|
- - ">="
|
18
18
|
- !ruby/object:Gem::Version
|
19
|
-
version: 0.
|
19
|
+
version: 0.14.0
|
20
20
|
type: :runtime
|
21
21
|
prerelease: false
|
22
22
|
version_requirements: !ruby/object:Gem::Requirement
|
23
23
|
requirements:
|
24
24
|
- - ">="
|
25
25
|
- !ruby/object:Gem::Version
|
26
|
-
version: 0.
|
26
|
+
version: 0.14.0
|
27
27
|
- !ruby/object:Gem::Dependency
|
28
28
|
name: connection_pool
|
29
29
|
requirement: !ruby/object:Gem::Requirement
|
@@ -207,14 +207,7 @@ metadata:
|
|
207
207
|
changelog_uri: https://github.com/sidekiq/sidekiq/blob/main/Changes.md
|
208
208
|
source_code_uri: https://github.com/sidekiq/sidekiq
|
209
209
|
rubygems_mfa_required: 'true'
|
210
|
-
post_install_message:
|
211
|
-
|
212
|
-
Welcome to Sidekiq 7.0!
|
213
|
-
|
214
|
-
1. Use `gem 'sidekiq', '<7'` in your Gemfile if you don't want this new version.
|
215
|
-
2. Read the release notes at https://github.com/sidekiq/sidekiq/blob/main/docs/7.0-Upgrade.md
|
216
|
-
3. If you have problems, search for open/closed issues at https://github.com/sidekiq/sidekiq/issues/
|
217
|
-
|
210
|
+
post_install_message:
|
218
211
|
rdoc_options: []
|
219
212
|
require_paths:
|
220
213
|
- lib
|
@@ -234,4 +227,3 @@ signing_key:
|
|
234
227
|
specification_version: 4
|
235
228
|
summary: Simple, efficient background processing for Ruby
|
236
229
|
test_files: []
|
237
|
-
...
|