sidekiq 6.2.2 → 6.4.0

Sign up to get free protection for your applications and to get access to all the features.

Potentially problematic release.


This version of sidekiq might be problematic. Click here for more details.

Files changed (48) hide show
  1. checksums.yaml +4 -4
  2. data/Changes.md +69 -1
  3. data/LICENSE +3 -3
  4. data/README.md +3 -3
  5. data/lib/generators/sidekiq/job_generator.rb +57 -0
  6. data/lib/generators/sidekiq/templates/{worker.rb.erb → job.rb.erb} +2 -2
  7. data/lib/generators/sidekiq/templates/{worker_spec.rb.erb → job_spec.rb.erb} +1 -1
  8. data/lib/generators/sidekiq/templates/{worker_test.rb.erb → job_test.rb.erb} +1 -1
  9. data/lib/sidekiq/api.rb +7 -4
  10. data/lib/sidekiq/cli.rb +13 -3
  11. data/lib/sidekiq/client.rb +5 -39
  12. data/lib/sidekiq/delay.rb +2 -0
  13. data/lib/sidekiq/extensions/action_mailer.rb +2 -2
  14. data/lib/sidekiq/extensions/active_record.rb +2 -2
  15. data/lib/sidekiq/extensions/class_methods.rb +2 -2
  16. data/lib/sidekiq/extensions/generic_proxy.rb +2 -2
  17. data/lib/sidekiq/fetch.rb +4 -3
  18. data/lib/sidekiq/job.rb +8 -3
  19. data/lib/sidekiq/job_retry.rb +6 -4
  20. data/lib/sidekiq/job_util.rb +65 -0
  21. data/lib/sidekiq/launcher.rb +5 -1
  22. data/lib/sidekiq/manager.rb +7 -9
  23. data/lib/sidekiq/middleware/current_attributes.rb +57 -0
  24. data/lib/sidekiq/rails.rb +11 -0
  25. data/lib/sidekiq/redis_connection.rb +4 -6
  26. data/lib/sidekiq/scheduled.rb +44 -15
  27. data/lib/sidekiq/util.rb +13 -0
  28. data/lib/sidekiq/version.rb +1 -1
  29. data/lib/sidekiq/web/application.rb +7 -4
  30. data/lib/sidekiq/web/helpers.rb +1 -12
  31. data/lib/sidekiq/worker.rb +127 -7
  32. data/lib/sidekiq.rb +8 -1
  33. data/sidekiq.gemspec +1 -1
  34. data/web/assets/javascripts/application.js +82 -61
  35. data/web/assets/javascripts/dashboard.js +51 -51
  36. data/web/assets/stylesheets/application-dark.css +19 -23
  37. data/web/assets/stylesheets/application-rtl.css +0 -4
  38. data/web/assets/stylesheets/application.css +10 -108
  39. data/web/locales/en.yml +1 -1
  40. data/web/views/_footer.erb +1 -1
  41. data/web/views/_poll_link.erb +2 -5
  42. data/web/views/_summary.erb +7 -7
  43. data/web/views/dashboard.erb +8 -8
  44. data/web/views/layout.erb +1 -1
  45. data/web/views/queue.erb +10 -10
  46. data/web/views/queues.erb +1 -1
  47. metadata +9 -7
  48. data/lib/generators/sidekiq/worker_generator.rb +0 -57
data/lib/sidekiq/rails.rb CHANGED
@@ -37,6 +37,17 @@ module Sidekiq
37
37
  end
38
38
  end
39
39
 
40
+ initializer "sidekiq.rails_logger" do
41
+ Sidekiq.configure_server do |_|
42
+ # This is the integration code necessary so that if code uses `Rails.logger.info "Hello"`,
43
+ # it will appear in the Sidekiq console with all of the job context. See #5021 and
44
+ # https://github.com/rails/rails/blob/b5f2b550f69a99336482739000c58e4e04e033aa/railties/lib/rails/commands/server/server_command.rb#L82-L84
45
+ unless ::Rails.logger == ::Sidekiq.logger || ::ActiveSupport::Logger.logger_outputs_to?(::Rails.logger, $stdout)
46
+ ::Rails.logger.extend(::ActiveSupport::Logger.broadcast(::Sidekiq.logger))
47
+ end
48
+ end
49
+ end
50
+
40
51
  # This hook happens after all initializers are run, just before returning
41
52
  # from config/environment.rb back to sidekiq/cli.rb.
42
53
  #
@@ -94,12 +94,10 @@ module Sidekiq
94
94
  def log_info(options)
95
95
  redacted = "REDACTED"
96
96
 
97
- # deep clone so we can muck with these options all we want
98
- #
99
- # exclude SSL params from dump-and-load because some information isn't
100
- # safely dumpable in current Rubies
101
- keys = options.keys
102
- keys.delete(:ssl_params)
97
+ # Deep clone so we can muck with these options all we want and exclude
98
+ # params from dump-and-load that may contain objects that Marshal is
99
+ # unable to safely dump.
100
+ keys = options.keys - [:logger, :ssl_params]
103
101
  scrubbed_options = Marshal.load(Marshal.dump(options.slice(*keys)))
104
102
  if scrubbed_options[:url] && (uri = URI.parse(scrubbed_options[:url])) && uri.password
105
103
  uri.password = redacted
@@ -9,29 +9,56 @@ module Sidekiq
9
9
  SETS = %w[retry schedule]
10
10
 
11
11
  class Enq
12
- def enqueue_jobs(now = Time.now.to_f.to_s, sorted_sets = SETS)
12
+ LUA_ZPOPBYSCORE = <<~LUA
13
+ local key, now = KEYS[1], ARGV[1]
14
+ local jobs = redis.call("zrangebyscore", key, "-inf", now, "limit", 0, 1)
15
+ if jobs[1] then
16
+ redis.call("zrem", key, jobs[1])
17
+ return jobs[1]
18
+ end
19
+ LUA
20
+
21
+ def initialize
22
+ @done = false
23
+ @lua_zpopbyscore_sha = nil
24
+ end
25
+
26
+ def enqueue_jobs(sorted_sets = SETS)
13
27
  # A job's "score" in Redis is the time at which it should be processed.
14
28
  # Just check Redis for the set of jobs with a timestamp before now.
15
29
  Sidekiq.redis do |conn|
16
30
  sorted_sets.each do |sorted_set|
17
- # Get next items in the queue with scores (time to execute) <= now.
18
- until (jobs = conn.zrangebyscore(sorted_set, "-inf", now, limit: [0, 100])).empty?
19
- # We need to go through the list one at a time to reduce the risk of something
20
- # going wrong between the time jobs are popped from the scheduled queue and when
21
- # they are pushed onto a work queue and losing the jobs.
22
- jobs.each do |job|
23
- # Pop item off the queue and add it to the work queue. If the job can't be popped from
24
- # the queue, it's because another process already popped it so we can move on to the
25
- # next one.
26
- if conn.zrem(sorted_set, job)
27
- Sidekiq::Client.push(Sidekiq.load_json(job))
28
- Sidekiq.logger.debug { "enqueued #{sorted_set}: #{job}" }
29
- end
30
- end
31
+ # Get next item in the queue with score (time to execute) <= now.
32
+ # We need to go through the list one at a time to reduce the risk of something
33
+ # going wrong between the time jobs are popped from the scheduled queue and when
34
+ # they are pushed onto a work queue and losing the jobs.
35
+ while !@done && (job = zpopbyscore(conn, keys: [sorted_set], argv: [Time.now.to_f.to_s]))
36
+ Sidekiq::Client.push(Sidekiq.load_json(job))
37
+ Sidekiq.logger.debug { "enqueued #{sorted_set}: #{job}" }
31
38
  end
32
39
  end
33
40
  end
34
41
  end
42
+
43
+ def terminate
44
+ @done = true
45
+ end
46
+
47
+ private
48
+
49
+ def zpopbyscore(conn, keys: nil, argv: nil)
50
+ if @lua_zpopbyscore_sha.nil?
51
+ raw_conn = conn.respond_to?(:redis) ? conn.redis : conn
52
+ @lua_zpopbyscore_sha = raw_conn.script(:load, LUA_ZPOPBYSCORE)
53
+ end
54
+
55
+ conn.evalsha(@lua_zpopbyscore_sha, keys: keys, argv: argv)
56
+ rescue Redis::CommandError => e
57
+ raise unless e.message.start_with?("NOSCRIPT")
58
+
59
+ @lua_zpopbyscore_sha = nil
60
+ retry
61
+ end
35
62
  end
36
63
 
37
64
  ##
@@ -55,6 +82,8 @@ module Sidekiq
55
82
  # Shut down this instance, will pause until the thread is dead.
56
83
  def terminate
57
84
  @done = true
85
+ @enq.terminate if @enq.respond_to?(:terminate)
86
+
58
87
  if @thread
59
88
  t = @thread
60
89
  @thread = nil
data/lib/sidekiq/util.rb CHANGED
@@ -39,6 +39,19 @@ module Sidekiq
39
39
  module Util
40
40
  include ExceptionHandler
41
41
 
42
+ # hack for quicker development / testing environment #2774
43
+ PAUSE_TIME = $stdout.tty? ? 0.1 : 0.5
44
+
45
+ # Wait for the orblock to be true or the deadline passed.
46
+ def wait_for(deadline, &condblock)
47
+ remaining = deadline - ::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
48
+ while remaining > PAUSE_TIME
49
+ return if condblock.call
50
+ sleep PAUSE_TIME
51
+ remaining = deadline - ::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
52
+ end
53
+ end
54
+
42
55
  def watchdog(last_words)
43
56
  yield
44
57
  rescue Exception => ex
@@ -1,5 +1,5 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  module Sidekiq
4
- VERSION = "6.2.2"
4
+ VERSION = "6.4.0"
5
5
  end
@@ -50,7 +50,10 @@ module Sidekiq
50
50
 
51
51
  get "/" do
52
52
  @redis_info = redis_info.select { |k, v| REDIS_KEYS.include? k }
53
- stats_history = Sidekiq::Stats::History.new((params["days"] || 30).to_i)
53
+ days = (params["days"] || 30).to_i
54
+ return halt(401) if days < 1 || days > 180
55
+
56
+ stats_history = Sidekiq::Stats::History.new(days)
54
57
  @processed_history = stats_history.processed
55
58
  @failed_history = stats_history.failed
56
59
 
@@ -91,8 +94,8 @@ module Sidekiq
91
94
 
92
95
  @count = (params["count"] || 25).to_i
93
96
  @queue = Sidekiq::Queue.new(@name)
94
- (@current_page, @total_size, @messages) = page("queue:#{@name}", params["page"], @count, reverse: params["direction"] == "asc")
95
- @messages = @messages.map { |msg| Sidekiq::JobRecord.new(msg, @name) }
97
+ (@current_page, @total_size, @jobs) = page("queue:#{@name}", params["page"], @count, reverse: params["direction"] == "asc")
98
+ @jobs = @jobs.map { |msg| Sidekiq::JobRecord.new(msg, @name) }
96
99
 
97
100
  erb(:queue)
98
101
  end
@@ -299,7 +302,7 @@ module Sidekiq
299
302
  return [404, {"Content-Type" => "text/plain", "X-Cascade" => "pass"}, ["Not Found"]] unless action
300
303
 
301
304
  app = @klass
302
- resp = catch(:halt) do # rubocop:disable Standard/SemanticBlocks
305
+ resp = catch(:halt) do
303
306
  self.class.run_befores(app, action)
304
307
  action.instance_exec env, &action.block
305
308
  ensure
@@ -70,17 +70,6 @@ module Sidekiq
70
70
  @head_html.join if defined?(@head_html)
71
71
  end
72
72
 
73
- def poll_path
74
- if current_path != "" && params["poll"]
75
- path = root_path + current_path
76
- query_string = to_query_string(params.slice(*params.keys - %w[page poll]))
77
- path += "?#{query_string}" unless query_string.empty?
78
- path
79
- else
80
- ""
81
- end
82
- end
83
-
84
73
  def text_direction
85
74
  get_locale["TextDirection"] || "ltr"
86
75
  end
@@ -203,7 +192,7 @@ module Sidekiq
203
192
  [score.to_f, jid]
204
193
  end
205
194
 
206
- SAFE_QPARAMS = %w[page poll direction]
195
+ SAFE_QPARAMS = %w[page direction]
207
196
 
208
197
  # Merge options with current params, filter safe params, and stringify to query string
209
198
  def qparams(options)
@@ -9,6 +9,7 @@ module Sidekiq
9
9
  #
10
10
  # class HardWorker
11
11
  # include Sidekiq::Worker
12
+ # sidekiq_options queue: 'critical', retry: 5
12
13
  #
13
14
  # def perform(*args)
14
15
  # # do some work
@@ -20,6 +21,26 @@ module Sidekiq
20
21
  # HardWorker.perform_async(1, 2, 3)
21
22
  #
22
23
  # Note that perform_async is a class method, perform is an instance method.
24
+ #
25
+ # Sidekiq::Worker also includes several APIs to provide compatibility with
26
+ # ActiveJob.
27
+ #
28
+ # class SomeWorker
29
+ # include Sidekiq::Worker
30
+ # queue_as :critical
31
+ #
32
+ # def perform(...)
33
+ # end
34
+ # end
35
+ #
36
+ # SomeWorker.set(wait_until: 1.hour).perform_async(123)
37
+ #
38
+ # Note that arguments passed to the job must still obey Sidekiq's
39
+ # best practice for simple, JSON-native data types. Sidekiq will not
40
+ # implement ActiveJob's more complex argument serialization. For
41
+ # this reason, we don't implement `perform_later` as our call semantics
42
+ # are very different.
43
+ #
23
44
  module Worker
24
45
  ##
25
46
  # The Options module is extracted so we can include it in ActiveJob::Base
@@ -150,33 +171,95 @@ module Sidekiq
150
171
  # SomeWorker.set(queue: 'foo').perform_async(....)
151
172
  #
152
173
  class Setter
174
+ include Sidekiq::JobUtil
175
+
153
176
  def initialize(klass, opts)
154
177
  @klass = klass
155
178
  @opts = opts
179
+
180
+ # ActiveJob compatibility
181
+ interval = @opts.delete(:wait_until) || @opts.delete(:wait)
182
+ at(interval) if interval
156
183
  end
157
184
 
158
185
  def set(options)
186
+ interval = options.delete(:wait_until) || options.delete(:wait)
159
187
  @opts.merge!(options)
188
+ at(interval) if interval
160
189
  self
161
190
  end
162
191
 
163
192
  def perform_async(*args)
164
- @klass.client_push(@opts.merge("args" => args, "class" => @klass))
193
+ if @opts["sync"] == true
194
+ perform_inline(*args)
195
+ else
196
+ @klass.client_push(@opts.merge("args" => args, "class" => @klass))
197
+ end
198
+ end
199
+
200
+ # Explicit inline execution of a job. Returns nil if the job did not
201
+ # execute, true otherwise.
202
+ def perform_inline(*args)
203
+ raw = @opts.merge("args" => args, "class" => @klass).transform_keys(&:to_s)
204
+
205
+ # validate and normalize payload
206
+ item = normalize_item(raw)
207
+ queue = item["queue"]
208
+
209
+ # run client-side middleware
210
+ result = Sidekiq.client_middleware.invoke(item["class"], item, queue, Sidekiq.redis_pool) do
211
+ item
212
+ end
213
+ return nil unless result
214
+
215
+ # round-trip the payload via JSON
216
+ msg = Sidekiq.load_json(Sidekiq.dump_json(item))
217
+
218
+ # prepare the job instance
219
+ klass = msg["class"].constantize
220
+ job = klass.new
221
+ job.jid = msg["jid"]
222
+ job.bid = msg["bid"] if job.respond_to?(:bid)
223
+
224
+ # run the job through server-side middleware
225
+ result = Sidekiq.server_middleware.invoke(job, msg, msg["queue"]) do
226
+ # perform it
227
+ job.perform(*msg["args"])
228
+ true
229
+ end
230
+ return nil unless result
231
+ # jobs do not return a result. they should store any
232
+ # modified state.
233
+ true
234
+ end
235
+ alias_method :perform_sync, :perform_inline
236
+
237
+ def perform_bulk(args, batch_size: 1_000)
238
+ hash = @opts.transform_keys(&:to_s)
239
+ result = args.each_slice(batch_size).flat_map do |slice|
240
+ Sidekiq::Client.push_bulk(hash.merge("class" => @klass, "args" => slice))
241
+ end
242
+
243
+ result.is_a?(Enumerator::Lazy) ? result.force : result
165
244
  end
166
245
 
167
246
  # +interval+ must be a timestamp, numeric or something that acts
168
247
  # numeric (like an activesupport time interval).
169
248
  def perform_in(interval, *args)
249
+ at(interval).perform_async(*args)
250
+ end
251
+ alias_method :perform_at, :perform_in
252
+
253
+ private
254
+
255
+ def at(interval)
170
256
  int = interval.to_f
171
257
  now = Time.now.to_f
172
258
  ts = (int < 1_000_000_000 ? now + int : int)
173
-
174
- payload = @opts.merge("class" => @klass, "args" => args)
175
259
  # Optimization to enqueue something now that is scheduled to go out now or in the past
176
- payload["at"] = ts if ts > now
177
- @klass.client_push(payload)
260
+ @opts["at"] = ts if ts > now
261
+ self
178
262
  end
179
- alias_method :perform_at, :perform_in
180
263
  end
181
264
 
182
265
  module ClassMethods
@@ -192,12 +275,49 @@ module Sidekiq
192
275
  raise ArgumentError, "Do not call .delay_until on a Sidekiq::Worker class, call .perform_at"
193
276
  end
194
277
 
278
+ def queue_as(q)
279
+ sidekiq_options("queue" => q.to_s)
280
+ end
281
+
195
282
  def set(options)
196
283
  Setter.new(self, options)
197
284
  end
198
285
 
199
286
  def perform_async(*args)
200
- client_push("class" => self, "args" => args)
287
+ Setter.new(self, {}).perform_async(*args)
288
+ end
289
+
290
+ # Inline execution of job's perform method after passing through Sidekiq.client_middleware and Sidekiq.server_middleware
291
+ def perform_inline(*args)
292
+ Setter.new(self, {}).perform_inline(*args)
293
+ end
294
+
295
+ ##
296
+ # Push a large number of jobs to Redis, while limiting the batch of
297
+ # each job payload to 1,000. This method helps cut down on the number
298
+ # of round trips to Redis, which can increase the performance of enqueueing
299
+ # large numbers of jobs.
300
+ #
301
+ # +items+ must be an Array of Arrays.
302
+ #
303
+ # For finer-grained control, use `Sidekiq::Client.push_bulk` directly.
304
+ #
305
+ # Example (3 Redis round trips):
306
+ #
307
+ # SomeWorker.perform_async(1)
308
+ # SomeWorker.perform_async(2)
309
+ # SomeWorker.perform_async(3)
310
+ #
311
+ # Would instead become (1 Redis round trip):
312
+ #
313
+ # SomeWorker.perform_bulk([[1], [2], [3]])
314
+ #
315
+ def perform_bulk(items, batch_size: 1_000)
316
+ result = items.each_slice(batch_size).flat_map do |slice|
317
+ Sidekiq::Client.push_bulk("class" => self, "args" => slice)
318
+ end
319
+
320
+ result.is_a?(Enumerator::Lazy) ? result.force : result
201
321
  end
202
322
 
203
323
  # +interval+ must be a timestamp, numeric or something that acts
data/lib/sidekiq.rb CHANGED
@@ -6,6 +6,7 @@ fail "Sidekiq #{Sidekiq::VERSION} does not support Ruby versions below 2.5.0." i
6
6
  require "sidekiq/logger"
7
7
  require "sidekiq/client"
8
8
  require "sidekiq/worker"
9
+ require "sidekiq/job"
9
10
  require "sidekiq/redis_connection"
10
11
  require "sidekiq/delay"
11
12
 
@@ -25,6 +26,7 @@ module Sidekiq
25
26
  timeout: 25,
26
27
  poll_interval_average: nil,
27
28
  average_scheduled_poll_interval: 5,
29
+ on_complex_arguments: :warn,
28
30
  error_handlers: [],
29
31
  death_handlers: [],
30
32
  lifecycle_events: {
@@ -100,7 +102,8 @@ module Sidekiq
100
102
  # 2550 Failover can cause the server to become a replica, need
101
103
  # to disconnect and reopen the socket to get back to the primary.
102
104
  # 4495 Use the same logic if we have a "Not enough replicas" error from the primary
103
- if retryable && ex.message =~ /READONLY|NOREPLICAS/
105
+ # 4985 Use the same logic when a blocking command is force-unblocked
106
+ if retryable && ex.message =~ /READONLY|NOREPLICAS|UNBLOCKED/
104
107
  conn.disconnect!
105
108
  retryable = false
106
109
  retry
@@ -250,6 +253,10 @@ module Sidekiq
250
253
  options[:lifecycle_events][event] << block
251
254
  end
252
255
 
256
+ def self.strict_args!(mode = :raise)
257
+ options[:on_complex_arguments] = mode
258
+ end
259
+
253
260
  # We are shutting down Sidekiq but what about workers that
254
261
  # are working on some long job? This error is
255
262
  # raised in workers that have not finished within the hard
data/sidekiq.gemspec CHANGED
@@ -18,7 +18,7 @@ Gem::Specification.new do |gem|
18
18
  "homepage_uri" => "https://sidekiq.org",
19
19
  "bug_tracker_uri" => "https://github.com/mperham/sidekiq/issues",
20
20
  "documentation_uri" => "https://github.com/mperham/sidekiq/wiki",
21
- "changelog_uri" => "https://github.com/mperham/sidekiq/blob/master/Changes.md",
21
+ "changelog_uri" => "https://github.com/mperham/sidekiq/blob/main/Changes.md",
22
22
  "source_code_uri" => "https://github.com/mperham/sidekiq"
23
23
  }
24
24