sidekiq 7.0.8 → 7.1.4

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 69b692f7976998a1655a5c6f108c0a1f686fdcdcde164f6cde071f9ea3f89ced
4
- data.tar.gz: d78d581fa48b744789b3af117a55d71bab1037b592f58ee9c74cf2c132716e0c
3
+ metadata.gz: 0a6064918f9c33be1d21f890b9cff080969fb44a916fe67ccb36958fe0a3b8f3
4
+ data.tar.gz: baf268f21f27e0dac2fc287f247910afe493e4dcaee21aa769ccebbed03e7699
5
5
  SHA512:
6
- metadata.gz: 42d16710f20a67a94df6498cf1fb5097a8795a5611f058e58d60e43e82503d26871c6d3061b3404590a0e0eba319997cb7e1daec3a88b66c2356e9cc0164a781
7
- data.tar.gz: 5b6d9aa7512a67cb552c3a1bed37aa111a877927e7690f53efd3e85aa1ea7049fb75eb05b31232810c8a805f95cb0959a54cc80bafb5c80f9aae0c0701d158b7
6
+ metadata.gz: c8f4e3caaeab143f20fdd592dc7939f09a302642867bddb4fcd02cbbaa0e734d7685ee3ad02543d3bb3ba6cbedaf2cbab3577a849c03d2f351aa7918d6eab73f
7
+ data.tar.gz: 1cf71898600afb872ee717be6496da7a146aa8e2f528b6c73e3630a64f71869bc23bfa2d7130f021c7f5e12b473ed3b537cebb4a8fedcfbe517af48dde5bbea6
data/Changes.md CHANGED
@@ -2,6 +2,60 @@
2
2
 
3
3
  [Sidekiq Changes](https://github.com/sidekiq/sidekiq/blob/main/Changes.md) | [Sidekiq Pro Changes](https://github.com/sidekiq/sidekiq/blob/main/Pro-Changes.md) | [Sidekiq Enterprise Changes](https://github.com/sidekiq/sidekiq/blob/main/Ent-Changes.md)
4
4
 
5
+ 7.1.4
6
+ ----------
7
+
8
+ - Fix empty `retry_for` logic [#6035]
9
+
10
+ 7.1.3
11
+ ----------
12
+
13
+ - Add `sidekiq_options retry_for: 48.hours` to allow time-based retry windows [#6029]
14
+ - Support sidekiq_retry_in and sidekiq_retries_exhausted_block in ActiveJobs (#5994)
15
+ - Lowercase all Rack headers for Rack 3.0 [#5951]
16
+ - Validate Sidekiq::Web page refresh delay to avoid potential DoS,
17
+ CVE-2023-26141, thanks for reporting Keegan!
18
+
19
+ 7.1.2
20
+ ----------
21
+
22
+ - Mark Web UI assets as private so CDNs won't cache them [#5936]
23
+ - Fix stackoverflow when using Oj and the JSON log formatter [#5920]
24
+ - Remove spurious `enqueued_at` from scheduled ActiveJobs [#5937]
25
+
26
+ 7.1.1
27
+ ----------
28
+
29
+ - Support multiple CurrentAttributes [#5904]
30
+ - Speed up latency fetch with large queues on Redis <7 [#5910]
31
+ - Allow a larger default client pool [#5886]
32
+ - Ensure Sidekiq.options[:environment] == RAILS_ENV [#5932]
33
+
34
+ 7.1.0
35
+ ----------
36
+
37
+ - Improve display of ActiveJob arguments in Web UI [#5825, cover]
38
+ - Update `push_bulk` to push `batch_size` jobs at a time and allow laziness [#5827, fatkodima]
39
+ This allows Sidekiq::Client to push unlimited jobs as long as it has enough memory for the batch_size.
40
+ - Update `perform_bulk` to use `push_bulk` internally.
41
+ - Change return value of `push_bulk` to map 1-to-1 with arguments.
42
+ If you call `push_bulk(args: [[1], [2], [3]])`, you will now always get
43
+ an array of 3 values as the result: `["jid1", nil, "jid3"]` where nil means
44
+ that particular job did not push successfully (possibly due to middleware
45
+ stopping it). Previously nil values were removed so it was impossible to tell
46
+ which jobs pushed successfully and which did not.
47
+ - Migrate away from all deprecated Redis commands [#5788]
48
+ Sidekiq will now print a warning if you use one of those deprecated commands.
49
+ - Prefix all Sidekiq thread names [#5872]
50
+
51
+ 7.0.9
52
+ ----------
53
+
54
+ - Restore confirmation dialogs in Web UI [#5881, shevaun]
55
+ - Increase fetch timeout to minimize ReadTimeoutError [#5874]
56
+ - Reverse histogram tooltip ordering [#5868]
57
+ - Add Scottish Gaelic (gd) locale [#5867, GunChleoc]
58
+
5
59
  7.0.8
6
60
  ----------
7
61
 
@@ -76,6 +130,11 @@ end
76
130
  - Job Execution metrics!!!
77
131
  - See `docs/7.0-Upgrade.md` for release notes
78
132
 
133
+ 6.5.9
134
+ ----------
135
+
136
+ - Ensure Sidekiq.options[:environment] == RAILS_ENV [#5932]
137
+
79
138
  6.5.8
80
139
  ----------
81
140
 
data/bin/sidekiqload CHANGED
@@ -1,5 +1,23 @@
1
1
  #!/usr/bin/env ruby
2
2
 
3
+ #
4
+ # bin/sidekiqload is a helpful script to load test and
5
+ # performance tune Sidekiq's core. It creates 500,000 no-op
6
+ # jobs and executes them as fast as possible.
7
+ # Example Usage:
8
+ #
9
+ # > RUBY_YJIT_ENABLE=1 LATENCY=0 THREADS=10 bin/sidekiqload
10
+ # Result: Done, 500000 jobs in 20.264945 sec, 24673 jobs/sec
11
+ #
12
+ # Use LATENCY=1 to get a more real world network setup
13
+ # but you'll need to setup and start toxiproxy as noted below.
14
+ #
15
+ # Use AJ=1 to test ActiveJob instead of plain old Sidekiq::Jobs so
16
+ # you can see the runtime performance difference between the two APIs.
17
+ #
18
+ # None of this script is considered a public API and may change over time.
19
+ #
20
+
3
21
  # Quiet some warnings we see when running in warning mode:
4
22
  # RUBYOPT=-w bundle exec sidekiq
5
23
  $TESTING = false
@@ -32,7 +50,7 @@ if ENV["AJ"]
32
50
  ActiveJob::Base.logger.level = Logger::WARN
33
51
 
34
52
  class LoadJob < ActiveJob::Base
35
- def perform(idx, ts=nil)
53
+ def perform(idx, ts = nil)
36
54
  puts(Time.now.to_f - ts) if !ts.nil?
37
55
  end
38
56
  end
@@ -219,11 +237,11 @@ end
219
237
  ll = Loader.new
220
238
  ll.configure
221
239
 
222
- unless ENV["GC"] || ENV["PROFILE"]
240
+ if ENV["WARM"]
223
241
  ll.setup
224
242
  ll.run("warmup")
225
243
  end
226
244
 
227
245
  ll.setup
228
- ll.run("ideal")
246
+ ll.run("load")
229
247
  ll.done
data/lib/sidekiq/api.rb CHANGED
@@ -92,11 +92,11 @@ module Sidekiq
92
92
  pipeline.zcard("retry")
93
93
  pipeline.zcard("dead")
94
94
  pipeline.scard("processes")
95
- pipeline.lrange("queue:default", -1, -1)
95
+ pipeline.lindex("queue:default", -1)
96
96
  end
97
97
  }
98
98
 
99
- default_queue_latency = if (entry = pipe1_res[6].first)
99
+ default_queue_latency = if (entry = pipe1_res[6])
100
100
  job = begin
101
101
  Sidekiq.load_json(entry)
102
102
  rescue
@@ -264,8 +264,8 @@ module Sidekiq
264
264
  # @return [Float] in seconds
265
265
  def latency
266
266
  entry = Sidekiq.redis { |conn|
267
- conn.lrange(@rname, -1, -1)
268
- }.first
267
+ conn.lindex(@rname, -1)
268
+ }
269
269
  return 0 unless entry
270
270
  job = Sidekiq.load_json(entry)
271
271
  now = Time.now.to_f
@@ -391,13 +391,13 @@ module Sidekiq
391
391
  def display_args
392
392
  # Unwrap known wrappers so they show up in a human-friendly manner in the Web UI
393
393
  @display_args ||= if klass == "ActiveJob::QueueAdapters::SidekiqAdapter::JobWrapper"
394
- job_args = self["wrapped"] ? args[0]["arguments"] : []
394
+ job_args = self["wrapped"] ? deserialize_argument(args[0]["arguments"]) : []
395
395
  if (self["wrapped"] || args[0]) == "ActionMailer::DeliveryJob"
396
396
  # remove MailerClass, mailer_method and 'deliver_now'
397
397
  job_args.drop(3)
398
398
  elsif (self["wrapped"] || args[0]) == "ActionMailer::MailDeliveryJob"
399
399
  # remove MailerClass, mailer_method and 'deliver_now'
400
- job_args.drop(3).first["args"]
400
+ job_args.drop(3).first.values_at("params", "args")
401
401
  else
402
402
  job_args
403
403
  end
@@ -467,6 +467,29 @@ module Sidekiq
467
467
 
468
468
  private
469
469
 
470
+ ACTIVE_JOB_PREFIX = "_aj_"
471
+ GLOBALID_KEY = "_aj_globalid"
472
+
473
+ def deserialize_argument(argument)
474
+ case argument
475
+ when Array
476
+ argument.map { |arg| deserialize_argument(arg) }
477
+ when Hash
478
+ if serialized_global_id?(argument)
479
+ argument[GLOBALID_KEY]
480
+ else
481
+ argument.transform_values { |v| deserialize_argument(v) }
482
+ .reject { |k, _| k.start_with?(ACTIVE_JOB_PREFIX) }
483
+ end
484
+ else
485
+ argument
486
+ end
487
+ end
488
+
489
+ def serialized_global_id?(hash)
490
+ hash.size == 1 && hash.include?(GLOBALID_KEY)
491
+ end
492
+
470
493
  def uncompress_backtrace(backtrace)
471
494
  decoded = Base64.decode64(backtrace)
472
495
  uncompressed = Zlib::Inflate.inflate(decoded)
@@ -548,7 +571,7 @@ module Sidekiq
548
571
  def remove_job
549
572
  Sidekiq.redis do |conn|
550
573
  results = conn.multi { |transaction|
551
- transaction.zrangebyscore(parent.name, score, score)
574
+ transaction.zrange(parent.name, score, score, "BYSCORE")
552
575
  transaction.zremrangebyscore(parent.name, score, score)
553
576
  }.first
554
577
 
@@ -683,7 +706,7 @@ module Sidekiq
683
706
  end
684
707
 
685
708
  elements = Sidekiq.redis { |conn|
686
- conn.zrangebyscore(name, begin_score, end_score, withscores: true)
709
+ conn.zrange(name, begin_score, end_score, "BYSCORE", withscores: true)
687
710
  }
688
711
 
689
712
  elements.each_with_object([]) do |element, result|
@@ -724,7 +747,7 @@ module Sidekiq
724
747
  # @api private
725
748
  def delete_by_jid(score, jid)
726
749
  Sidekiq.redis do |conn|
727
- elements = conn.zrangebyscore(name, score, score)
750
+ elements = conn.zrange(name, score, score, "BYSCORE")
728
751
  elements.each do |element|
729
752
  if element.index(jid)
730
753
  message = Sidekiq.load_json(element)
data/lib/sidekiq/cli.rb CHANGED
@@ -230,6 +230,7 @@ module Sidekiq # :nodoc:
230
230
  # Both Sinatra 2.0+ and Sidekiq support this term.
231
231
  # RAILS_ENV and RACK_ENV are there for legacy support.
232
232
  @environment = cli_env || ENV["APP_ENV"] || ENV["RAILS_ENV"] || ENV["RACK_ENV"] || "development"
233
+ config[:environment] = @environment
233
234
  end
234
235
 
235
236
  def symbolize_keys_deep!(hash)
@@ -396,7 +397,7 @@ module Sidekiq # :nodoc:
396
397
  end
397
398
 
398
399
  def parse_config(path)
399
- erb = ERB.new(File.read(path))
400
+ erb = ERB.new(File.read(path), trim_mode: "-")
400
401
  erb.filename = File.expand_path(path)
401
402
  opts = YAML.safe_load(erb.result, permitted_classes: [Symbol], aliases: true) || {}
402
403
 
@@ -66,6 +66,7 @@ module Sidekiq
66
66
  # args - an array of simple arguments to the perform method, must be JSON-serializable
67
67
  # at - timestamp to schedule the job (optional), must be Numeric (e.g. Time.now.to_f)
68
68
  # retry - whether to retry this job if it fails, default true or an integer number of retries
69
+ # retry_for - relative amount of time to retry this job if it fails, default nil
69
70
  # backtrace - whether to save any error backtrace, default false
70
71
  #
71
72
  # If class is set to the class name, the jobs' options will be based on Sidekiq's default
@@ -96,8 +97,9 @@ module Sidekiq
96
97
 
97
98
  ##
98
99
  # Push a large number of jobs to Redis. This method cuts out the redis
99
- # network round trip latency. I wouldn't recommend pushing more than
100
- # 1000 per call but YMMV based on network quality, size of job args, etc.
100
+ # network round trip latency. It pushes jobs in batches if more than
101
+ # `:batch_size` (1000 by default) of jobs are passed. I wouldn't recommend making `:batch_size`
102
+ # larger than 1000 but YMMV based on network quality, size of job args, etc.
101
103
  # A large number of jobs can cause a bit of Redis command processing latency.
102
104
  #
103
105
  # Takes the same arguments as #push except that args is expected to be
@@ -105,13 +107,15 @@ module Sidekiq
105
107
  # is run through the client middleware pipeline and each job gets its own Job ID
106
108
  # as normal.
107
109
  #
108
- # Returns an array of the of pushed jobs' jids. The number of jobs pushed can be less
109
- # than the number given if the middleware stopped processing for one or more jobs.
110
+ # Returns an array of the of pushed jobs' jids, may contain nils if any client middleware
111
+ # prevented a job push.
112
+ #
113
+ # Example (pushing jobs in batches):
114
+ # push_bulk('class' => 'MyJob', 'args' => (1..100_000).to_a, batch_size: 1_000)
115
+ #
110
116
  def push_bulk(items)
117
+ batch_size = items.delete(:batch_size) || items.delete("batch_size") || 1_000
111
118
  args = items["args"]
112
- raise ArgumentError, "Bulk arguments must be an Array of Arrays: [[1], [2]]" unless args.is_a?(Array) && args.all?(Array)
113
- return [] if args.empty? # no jobs to push
114
-
115
119
  at = items.delete("at")
116
120
  raise ArgumentError, "Job 'at' must be a Numeric or an Array of Numeric timestamps" if at && (Array(at).empty? || !Array(at).all? { |entry| entry.is_a?(Numeric) })
117
121
  raise ArgumentError, "Job 'at' Array must have same size as 'args' Array" if at.is_a?(Array) && at.size != args.size
@@ -120,18 +124,26 @@ module Sidekiq
120
124
  raise ArgumentError, "Explicitly passing 'jid' when pushing more than one job is not supported" if jid && args.size > 1
121
125
 
122
126
  normed = normalize_item(items)
123
- payloads = args.map.with_index { |job_args, index|
124
- copy = normed.merge("args" => job_args, "jid" => SecureRandom.hex(12))
125
- copy["at"] = (at.is_a?(Array) ? at[index] : at) if at
126
- result = middleware.invoke(items["class"], copy, copy["queue"], @redis_pool) do
127
- verify_json(copy)
128
- copy
129
- end
130
- result || nil
131
- }.compact
127
+ result = args.each_slice(batch_size).flat_map do |slice|
128
+ raise ArgumentError, "Bulk arguments must be an Array of Arrays: [[1], [2]]" unless slice.is_a?(Array) && slice.all?(Array)
129
+ break [] if slice.empty? # no jobs to push
130
+
131
+ payloads = slice.map.with_index { |job_args, index|
132
+ copy = normed.merge("args" => job_args, "jid" => SecureRandom.hex(12))
133
+ copy["at"] = (at.is_a?(Array) ? at[index] : at) if at
134
+ result = middleware.invoke(items["class"], copy, copy["queue"], @redis_pool) do
135
+ verify_json(copy)
136
+ copy
137
+ end
138
+ result || nil
139
+ }
140
+
141
+ to_push = payloads.compact
142
+ raw_push(to_push) unless to_push.empty?
143
+ payloads.map { |payload| payload&.[]("jid") }
144
+ end
132
145
 
133
- raw_push(payloads) unless payloads.empty?
134
- payloads.collect { |payload| payload["jid"] }
146
+ result.is_a?(Enumerator::Lazy) ? result.force : result
135
147
  end
136
148
 
137
149
  # Allows sharding of jobs across any number of Redis instances. All jobs
@@ -160,8 +172,8 @@ module Sidekiq
160
172
  new.push(item)
161
173
  end
162
174
 
163
- def push_bulk(items)
164
- new.push_bulk(items)
175
+ def push_bulk(...)
176
+ new.push_bulk(...)
165
177
  end
166
178
 
167
179
  # Resque compatibility helpers. Note all helpers
@@ -235,6 +247,8 @@ module Sidekiq
235
247
  if payloads.first.key?("at")
236
248
  conn.zadd("schedule", payloads.flat_map { |hash|
237
249
  at = hash.delete("at").to_s
250
+ # ActiveJob sets this but the job has not been enqueued yet
251
+ hash.delete("enqueued_at")
238
252
  [at, Sidekiq.dump_json(hash)]
239
253
  })
240
254
  else
@@ -15,7 +15,7 @@ module Sidekiq
15
15
 
16
16
  def safe_thread(name, &block)
17
17
  Thread.new do
18
- Thread.current.name = name
18
+ Thread.current.name = "sidekiq.#{name}"
19
19
  watchdog(name, &block)
20
20
  end
21
21
  end
@@ -30,7 +30,8 @@ module Sidekiq
30
30
  },
31
31
  dead_max_jobs: 10_000,
32
32
  dead_timeout_in_seconds: 180 * 24 * 60 * 60, # 6 months
33
- reloader: proc { |&block| block.call }
33
+ reloader: proc { |&block| block.call },
34
+ backtrace_cleaner: ->(backtrace) { backtrace }
34
35
  }
35
36
 
36
37
  ERROR_HANDLER = ->(ex, ctx) {
@@ -38,7 +39,10 @@ module Sidekiq
38
39
  l = cfg.logger
39
40
  l.warn(Sidekiq.dump_json(ctx)) unless ctx.empty?
40
41
  l.warn("#{ex.class.name}: #{ex.message}")
41
- l.warn(ex.backtrace.join("\n")) unless ex.backtrace.nil?
42
+ unless ex.backtrace.nil?
43
+ backtrace = cfg[:backtrace_cleaner].call(ex.backtrace)
44
+ l.warn(backtrace.join("\n"))
45
+ end
42
46
  }
43
47
 
44
48
  def initialize(options = {})
@@ -52,6 +56,10 @@ module Sidekiq
52
56
  def_delegators :@options, :[], :[]=, :fetch, :key?, :has_key?, :merge!
53
57
  attr_reader :capsules
54
58
 
59
+ def to_json(*)
60
+ Sidekiq.dump_json(@options)
61
+ end
62
+
55
63
  # LEGACY: edits the default capsule
56
64
  # config.concurrency = 5
57
65
  def concurrency=(val)
@@ -123,7 +131,7 @@ module Sidekiq
123
131
  private def local_redis_pool
124
132
  # this is our internal client/housekeeping pool. each capsule has its
125
133
  # own pool for executing threads.
126
- @redis ||= new_redis_pool(5, "internal")
134
+ @redis ||= new_redis_pool(10, "internal")
127
135
  end
128
136
 
129
137
  def new_redis_pool(size, name = "unset")
@@ -259,7 +267,7 @@ module Sidekiq
259
267
  ctx[:_config] = self
260
268
  @options[:error_handlers].each do |handler|
261
269
  handler.call(ex, ctx)
262
- rescue => e
270
+ rescue Exception => e
263
271
  l = logger
264
272
  l.error "!!! ERROR HANDLER THREW AN ERROR !!!"
265
273
  l.error e
data/lib/sidekiq/fetch.rb CHANGED
@@ -44,7 +44,7 @@ module Sidekiq # :nodoc:
44
44
  return nil
45
45
  end
46
46
 
47
- queue, job = redis { |conn| conn.blocking_call(TIMEOUT + 1, "brpop", *qs, TIMEOUT) }
47
+ queue, job = redis { |conn| conn.blocking_call(conn.read_timeout + TIMEOUT, "brpop", *qs, TIMEOUT) }
48
48
  UnitOfWork.new(queue, job, config) if queue
49
49
  end
50
50
 
data/lib/sidekiq/job.rb CHANGED
@@ -239,11 +239,7 @@ module Sidekiq
239
239
 
240
240
  def perform_bulk(args, batch_size: 1_000)
241
241
  client = @klass.build_client
242
- result = args.each_slice(batch_size).flat_map do |slice|
243
- client.push_bulk(@opts.merge("class" => @klass, "args" => slice))
244
- end
245
-
246
- result.is_a?(Enumerator::Lazy) ? result.force : result
242
+ client.push_bulk(@opts.merge("class" => @klass, "args" => args, :batch_size => batch_size))
247
243
  end
248
244
 
249
245
  # +interval+ must be a timestamp, numeric or something that acts
@@ -71,6 +71,7 @@ module Sidekiq
71
71
  def initialize(capsule)
72
72
  @config = @capsule = capsule
73
73
  @max_retries = Sidekiq.default_configuration[:max_retries] || DEFAULT_MAX_RETRY_ATTEMPTS
74
+ @backtrace_cleaner = Sidekiq.default_configuration[:backtrace_cleaner]
74
75
  end
75
76
 
76
77
  # The global retry handler requires only the barest of data.
@@ -159,18 +160,21 @@ module Sidekiq
159
160
  end
160
161
 
161
162
  if msg["backtrace"]
163
+ backtrace = @backtrace_cleaner.call(exception.backtrace)
162
164
  lines = if msg["backtrace"] == true
163
- exception.backtrace
165
+ backtrace
164
166
  else
165
- exception.backtrace[0...msg["backtrace"].to_i]
167
+ backtrace[0...msg["backtrace"].to_i]
166
168
  end
167
169
 
168
170
  msg["error_backtrace"] = compress_backtrace(lines)
169
171
  end
170
172
 
171
- # Goodbye dear message, you (re)tried your best I'm sure.
172
173
  return retries_exhausted(jobinst, msg, exception) if count >= max_retry_attempts
173
174
 
175
+ rf = msg["retry_for"]
176
+ return retries_exhausted(jobinst, msg, exception) if rf && ((msg["failed_at"] + rf) < Time.now.to_f)
177
+
174
178
  strategy, delay = delay_for(jobinst, count, exception, msg)
175
179
  case strategy
176
180
  when :discard
@@ -195,7 +199,14 @@ module Sidekiq
195
199
  # sidekiq_retry_in can return two different things:
196
200
  # 1. When to retry next, as an integer of seconds
197
201
  # 2. A symbol which re-routes the job elsewhere, e.g. :discard, :kill, :default
198
- jobinst&.sidekiq_retry_in_block&.call(count, exception, msg)
202
+ block = jobinst&.sidekiq_retry_in_block
203
+
204
+ # the sidekiq_retry_in_block can be defined in a wrapped class (ActiveJob for instance)
205
+ unless msg["wrapped"].nil?
206
+ wrapped = Object.const_get(msg["wrapped"])
207
+ block = wrapped.respond_to?(:sidekiq_retry_in_block) ? wrapped.sidekiq_retry_in_block : nil
208
+ end
209
+ block&.call(count, exception, msg)
199
210
  rescue Exception => e
200
211
  handle_exception(e, {context: "Failure scheduling retry using the defined `sidekiq_retry_in` in #{jobinst.class.name}, falling back to default"})
201
212
  nil
@@ -217,6 +228,12 @@ module Sidekiq
217
228
  def retries_exhausted(jobinst, msg, exception)
218
229
  begin
219
230
  block = jobinst&.sidekiq_retries_exhausted_block
231
+
232
+ # the sidekiq_retries_exhausted_block can be defined in a wrapped class (ActiveJob for instance)
233
+ unless msg["wrapped"].nil?
234
+ wrapped = Object.const_get(msg["wrapped"])
235
+ block = wrapped.respond_to?(:sidekiq_retries_exhausted_block) ? wrapped.sidekiq_retries_exhausted_block : nil
236
+ end
220
237
  block&.call(msg, exception)
221
238
  rescue => e
222
239
  handle_exception(e, {context: "Error calling retries_exhausted", job: msg})
@@ -9,10 +9,11 @@ module Sidekiq
9
9
 
10
10
  def validate(item)
11
11
  raise(ArgumentError, "Job must be a Hash with 'class' and 'args' keys: `#{item}`") unless item.is_a?(Hash) && item.key?("class") && item.key?("args")
12
- raise(ArgumentError, "Job args must be an Array: `#{item}`") unless item["args"].is_a?(Array)
12
+ raise(ArgumentError, "Job args must be an Array: `#{item}`") unless item["args"].is_a?(Array) || item["args"].is_a?(Enumerator::Lazy)
13
13
  raise(ArgumentError, "Job class must be either a Class or String representation of the class name: `#{item}`") unless item["class"].is_a?(Class) || item["class"].is_a?(String)
14
14
  raise(ArgumentError, "Job 'at' must be a Numeric timestamp: `#{item}`") if item.key?("at") && !item["at"].is_a?(Numeric)
15
15
  raise(ArgumentError, "Job tags must be an Array: `#{item}`") if item["tags"] && !item["tags"].is_a?(Array)
16
+ raise(ArgumentError, "retry_for must be a relative amount of time, e.g. 48.hours `#{item}`") if item["retry_for"] && item["retry_for"] > 1_000_000_000
16
17
  end
17
18
 
18
19
  def verify_json(item)
@@ -24,7 +25,7 @@ module Sidekiq
24
25
  if (unsafe_item = json_unsafe?(args))
25
26
  msg = <<~EOM
26
27
  Job arguments to #{job_class} must be native JSON types, but #{unsafe_item.inspect} is a #{unsafe_item.class}.
27
- See https://github.com/sidekiq/sidekiq/wiki/Best-Practices.
28
+ See https://github.com/sidekiq/sidekiq/wiki/Best-Practices
28
29
  To disable this error, add `Sidekiq.strict_args!(false)` to your initializer.
29
30
  EOM
30
31
 
@@ -54,6 +55,7 @@ module Sidekiq
54
55
  item["jid"] ||= SecureRandom.hex(12)
55
56
  item["class"] = item["class"].to_s
56
57
  item["queue"] = item["queue"].to_s
58
+ item["retry_for"] = item["retry_for"].to_i if item["retry_for"]
57
59
  item["created_at"] ||= Time.now.to_f
58
60
  item
59
61
  end
@@ -166,7 +166,7 @@ module Sidekiq
166
166
  conn.multi { |transaction|
167
167
  transaction.sadd("processes", [key])
168
168
  transaction.exists(key)
169
- transaction.hmset(key, "info", to_json,
169
+ transaction.hset(key, "info", to_json,
170
170
  "busy", curstate.size,
171
171
  "beat", Time.now.to_f,
172
172
  "rtt_us", rtt,
@@ -70,7 +70,7 @@ module Sidekiq
70
70
  result.job_results[klass].add_metric "ms", time, ms.to_i if ms
71
71
  result.job_results[klass].add_metric "p", time, p.to_i if p
72
72
  result.job_results[klass].add_metric "f", time, f.to_i if f
73
- result.job_results[klass].add_hist time, Histogram.new(klass).fetch(conn, time)
73
+ result.job_results[klass].add_hist time, Histogram.new(klass).fetch(conn, time).reverse
74
74
  time -= 60
75
75
  end
76
76
  end
@@ -29,8 +29,8 @@ module Sidekiq
29
29
  1100, 1700, 2500, 3800, 5750,
30
30
  8500, 13000, 20000, 30000, 45000,
31
31
  65000, 100000, 150000, 225000, 335000,
32
- Float::INFINITY # the "maybe your job is too long" bucket
33
- ]
32
+ 1e20 # the "maybe your job is too long" bucket
33
+ ].freeze
34
34
  LABELS = [
35
35
  "20ms", "30ms", "45ms", "65ms", "100ms",
36
36
  "150ms", "225ms", "335ms", "500ms", "750ms",
@@ -38,7 +38,7 @@ module Sidekiq
38
38
  "8.5s", "13s", "20s", "30s", "45s",
39
39
  "65s", "100s", "150s", "225s", "335s",
40
40
  "Slow"
41
- ]
41
+ ].freeze
42
42
  FETCH = "GET u16 #0 GET u16 #1 GET u16 #2 GET u16 #3 \
43
43
  GET u16 #4 GET u16 #5 GET u16 #6 GET u16 #7 \
44
44
  GET u16 #8 GET u16 #9 GET u16 #10 GET u16 #11 \
@@ -73,7 +73,7 @@ module Sidekiq
73
73
  def fetch(conn, now = Time.now)
74
74
  window = now.utc.strftime("%d-%H:%-M")
75
75
  key = "#{@klass}-#{window}"
76
- conn.bitfield(key, *FETCH)
76
+ conn.bitfield_ro(key, *FETCH)
77
77
  end
78
78
 
79
79
  def persist(conn, now = Time.now)