sidekiq 6.0.7 → 6.4.2

Sign up to get free protection for your applications and to get access to all the features.

Potentially problematic release.


This version of sidekiq might be problematic. Click here for more details.

Files changed (101) hide show
  1. checksums.yaml +4 -4
  2. data/Changes.md +189 -2
  3. data/LICENSE +3 -3
  4. data/README.md +11 -10
  5. data/bin/sidekiq +8 -3
  6. data/bin/sidekiqload +57 -65
  7. data/bin/sidekiqmon +1 -1
  8. data/lib/generators/sidekiq/job_generator.rb +57 -0
  9. data/lib/generators/sidekiq/templates/{worker.rb.erb → job.rb.erb} +2 -2
  10. data/lib/generators/sidekiq/templates/{worker_spec.rb.erb → job_spec.rb.erb} +1 -1
  11. data/lib/generators/sidekiq/templates/{worker_test.rb.erb → job_test.rb.erb} +1 -1
  12. data/lib/sidekiq/api.rb +164 -116
  13. data/lib/sidekiq/cli.rb +49 -15
  14. data/lib/sidekiq/client.rb +51 -70
  15. data/lib/sidekiq/delay.rb +2 -0
  16. data/lib/sidekiq/extensions/action_mailer.rb +3 -2
  17. data/lib/sidekiq/extensions/active_record.rb +4 -3
  18. data/lib/sidekiq/extensions/class_methods.rb +5 -4
  19. data/lib/sidekiq/extensions/generic_proxy.rb +4 -2
  20. data/lib/sidekiq/fetch.rb +32 -23
  21. data/lib/sidekiq/job.rb +13 -0
  22. data/lib/sidekiq/job_logger.rb +16 -28
  23. data/lib/sidekiq/job_retry.rb +32 -33
  24. data/lib/sidekiq/job_util.rb +67 -0
  25. data/lib/sidekiq/launcher.rb +113 -54
  26. data/lib/sidekiq/logger.rb +11 -20
  27. data/lib/sidekiq/manager.rb +16 -18
  28. data/lib/sidekiq/middleware/chain.rb +10 -8
  29. data/lib/sidekiq/middleware/current_attributes.rb +57 -0
  30. data/lib/sidekiq/middleware/i18n.rb +4 -4
  31. data/lib/sidekiq/monitor.rb +1 -1
  32. data/lib/sidekiq/paginator.rb +8 -8
  33. data/lib/sidekiq/processor.rb +31 -31
  34. data/lib/sidekiq/rails.rb +36 -20
  35. data/lib/sidekiq/redis_connection.rb +16 -15
  36. data/lib/sidekiq/scheduled.rb +51 -16
  37. data/lib/sidekiq/sd_notify.rb +1 -1
  38. data/lib/sidekiq/testing/inline.rb +4 -4
  39. data/lib/sidekiq/testing.rb +38 -39
  40. data/lib/sidekiq/util.rb +41 -0
  41. data/lib/sidekiq/version.rb +1 -1
  42. data/lib/sidekiq/web/action.rb +2 -2
  43. data/lib/sidekiq/web/application.rb +21 -12
  44. data/lib/sidekiq/web/csrf_protection.rb +180 -0
  45. data/lib/sidekiq/web/helpers.rb +39 -33
  46. data/lib/sidekiq/web/router.rb +5 -2
  47. data/lib/sidekiq/web.rb +36 -72
  48. data/lib/sidekiq/worker.rb +135 -16
  49. data/lib/sidekiq.rb +33 -17
  50. data/sidekiq.gemspec +11 -4
  51. data/web/assets/images/apple-touch-icon.png +0 -0
  52. data/web/assets/javascripts/application.js +113 -65
  53. data/web/assets/javascripts/dashboard.js +51 -51
  54. data/web/assets/stylesheets/application-dark.css +64 -43
  55. data/web/assets/stylesheets/application-rtl.css +0 -4
  56. data/web/assets/stylesheets/application.css +42 -239
  57. data/web/locales/ar.yml +8 -2
  58. data/web/locales/en.yml +4 -1
  59. data/web/locales/es.yml +18 -2
  60. data/web/locales/fr.yml +8 -1
  61. data/web/locales/ja.yml +3 -0
  62. data/web/locales/lt.yml +1 -1
  63. data/web/locales/pl.yml +4 -4
  64. data/web/locales/ru.yml +4 -0
  65. data/web/views/_footer.erb +1 -1
  66. data/web/views/_job_info.erb +1 -1
  67. data/web/views/_poll_link.erb +2 -5
  68. data/web/views/_summary.erb +7 -7
  69. data/web/views/busy.erb +51 -20
  70. data/web/views/dashboard.erb +22 -14
  71. data/web/views/dead.erb +1 -1
  72. data/web/views/layout.erb +2 -1
  73. data/web/views/morgue.erb +6 -6
  74. data/web/views/queue.erb +11 -11
  75. data/web/views/queues.erb +4 -4
  76. data/web/views/retries.erb +7 -7
  77. data/web/views/retry.erb +1 -1
  78. data/web/views/scheduled.erb +1 -1
  79. metadata +24 -49
  80. data/.circleci/config.yml +0 -60
  81. data/.github/contributing.md +0 -32
  82. data/.github/issue_template.md +0 -11
  83. data/.gitignore +0 -13
  84. data/.standard.yml +0 -20
  85. data/3.0-Upgrade.md +0 -70
  86. data/4.0-Upgrade.md +0 -53
  87. data/5.0-Upgrade.md +0 -56
  88. data/6.0-Upgrade.md +0 -72
  89. data/COMM-LICENSE +0 -97
  90. data/Ent-2.0-Upgrade.md +0 -37
  91. data/Ent-Changes.md +0 -256
  92. data/Gemfile +0 -24
  93. data/Gemfile.lock +0 -208
  94. data/Pro-2.0-Upgrade.md +0 -138
  95. data/Pro-3.0-Upgrade.md +0 -44
  96. data/Pro-4.0-Upgrade.md +0 -35
  97. data/Pro-5.0-Upgrade.md +0 -25
  98. data/Pro-Changes.md +0 -782
  99. data/Rakefile +0 -10
  100. data/code_of_conduct.md +0 -50
  101. data/lib/generators/sidekiq/worker_generator.rb +0 -57
@@ -2,9 +2,12 @@
2
2
 
3
3
  require "securerandom"
4
4
  require "sidekiq/middleware/chain"
5
+ require "sidekiq/job_util"
5
6
 
6
7
  module Sidekiq
7
8
  class Client
9
+ include Sidekiq::JobUtil
10
+
8
11
  ##
9
12
  # Define client-side middleware:
10
13
  #
@@ -12,14 +15,14 @@ module Sidekiq
12
15
  # client.middleware do |chain|
13
16
  # chain.use MyClientMiddleware
14
17
  # end
15
- # client.push('class' => 'SomeWorker', 'args' => [1,2,3])
18
+ # client.push('class' => 'SomeJob', 'args' => [1,2,3])
16
19
  #
17
20
  # All client instances default to the globally-defined
18
21
  # Sidekiq.client_middleware but you can change as necessary.
19
22
  #
20
23
  def middleware(&block)
21
24
  @chain ||= Sidekiq.client_middleware
22
- if block_given?
25
+ if block
23
26
  @chain = @chain.dup
24
27
  yield @chain
25
28
  end
@@ -46,16 +49,16 @@ module Sidekiq
46
49
  # The main method used to push a job to Redis. Accepts a number of options:
47
50
  #
48
51
  # queue - the named queue to use, default 'default'
49
- # class - the worker class to call, required
52
+ # class - the job class to call, required
50
53
  # args - an array of simple arguments to the perform method, must be JSON-serializable
51
54
  # at - timestamp to schedule the job (optional), must be Numeric (e.g. Time.now.to_f)
52
55
  # retry - whether to retry this job if it fails, default true or an integer number of retries
53
56
  # backtrace - whether to save any error backtrace, default false
54
57
  #
55
58
  # If class is set to the class name, the jobs' options will be based on Sidekiq's default
56
- # worker options. Otherwise, they will be based on the job class's options.
59
+ # job options. Otherwise, they will be based on the job class's options.
57
60
  #
58
- # Any options valid for a worker class's sidekiq_options are also available here.
61
+ # Any options valid for a job class's sidekiq_options are also available here.
59
62
  #
60
63
  # All options must be strings, not symbols. NB: because we are serializing to JSON, all
61
64
  # symbols in 'args' will be converted to strings. Note that +backtrace: true+ can take quite a bit of
@@ -64,13 +67,15 @@ module Sidekiq
64
67
  # Returns a unique Job ID. If middleware stops the job, nil will be returned instead.
65
68
  #
66
69
  # Example:
67
- # push('queue' => 'my_queue', 'class' => MyWorker, 'args' => ['foo', 1, :bat => 'bar'])
70
+ # push('queue' => 'my_queue', 'class' => MyJob, 'args' => ['foo', 1, :bat => 'bar'])
68
71
  #
69
72
  def push(item)
70
73
  normed = normalize_item(item)
71
- payload = process_single(item["class"], normed)
72
-
74
+ payload = middleware.invoke(normed["class"], normed, normed["queue"], @redis_pool) do
75
+ normed
76
+ end
73
77
  if payload
78
+ verify_json(payload)
74
79
  raw_push([payload])
75
80
  payload["jid"]
76
81
  end
@@ -90,19 +95,25 @@ module Sidekiq
90
95
  # Returns an array of the of pushed jobs' jids. The number of jobs pushed can be less
91
96
  # than the number given if the middleware stopped processing for one or more jobs.
92
97
  def push_bulk(items)
93
- arg = items["args"].first
94
- return [] unless arg # no jobs to push
95
- raise ArgumentError, "Bulk arguments must be an Array of Arrays: [[1], [2]]" unless arg.is_a?(Array)
98
+ args = items["args"]
99
+ raise ArgumentError, "Bulk arguments must be an Array of Arrays: [[1], [2]]" unless args.is_a?(Array) && args.all?(Array)
100
+ return [] if args.empty? # no jobs to push
96
101
 
97
102
  at = items.delete("at")
98
- raise ArgumentError, "Job 'at' must be a Numeric or an Array of Numeric timestamps" if at && (Array(at).empty? || !Array(at).all?(Numeric))
103
+ raise ArgumentError, "Job 'at' must be a Numeric or an Array of Numeric timestamps" if at && (Array(at).empty? || !Array(at).all? { |entry| entry.is_a?(Numeric) })
104
+ raise ArgumentError, "Job 'at' Array must have same size as 'args' Array" if at.is_a?(Array) && at.size != args.size
105
+
106
+ jid = items.delete("jid")
107
+ raise ArgumentError, "Explicitly passing 'jid' when pushing more than one job is not supported" if jid && args.size > 1
99
108
 
100
109
  normed = normalize_item(items)
101
- payloads = items["args"].map.with_index { |args, index|
102
- copy = normed.merge("args" => args, "jid" => SecureRandom.hex(12), "enqueued_at" => Time.now.to_f)
110
+ payloads = args.map.with_index { |job_args, index|
111
+ copy = normed.merge("args" => job_args, "jid" => SecureRandom.hex(12))
103
112
  copy["at"] = (at.is_a?(Array) ? at[index] : at) if at
104
-
105
- result = process_single(items["class"], copy)
113
+ result = middleware.invoke(copy["class"], copy, copy["queue"], @redis_pool) do
114
+ verify_json(copy)
115
+ copy
116
+ end
106
117
  result || nil
107
118
  }.compact
108
119
 
@@ -115,8 +126,8 @@ module Sidekiq
115
126
  #
116
127
  # pool = ConnectionPool.new { Redis.new }
117
128
  # Sidekiq::Client.via(pool) do
118
- # SomeWorker.perform_async(1,2,3)
119
- # SomeOtherWorker.perform_async(1,2,3)
129
+ # SomeJob.perform_async(1,2,3)
130
+ # SomeOtherJob.perform_async(1,2,3)
120
131
  # end
121
132
  #
122
133
  # Generally this is only needed for very large Sidekiq installs processing
@@ -141,10 +152,10 @@ module Sidekiq
141
152
  end
142
153
 
143
154
  # Resque compatibility helpers. Note all helpers
144
- # should go through Worker#client_push.
155
+ # should go through Sidekiq::Job#client_push.
145
156
  #
146
157
  # Example usage:
147
- # Sidekiq::Client.enqueue(MyWorker, 'foo', 1, :bat => 'bar')
158
+ # Sidekiq::Client.enqueue(MyJob, 'foo', 1, :bat => 'bar')
148
159
  #
149
160
  # Messages are enqueued to the 'default' queue.
150
161
  #
@@ -153,14 +164,14 @@ module Sidekiq
153
164
  end
154
165
 
155
166
  # Example usage:
156
- # Sidekiq::Client.enqueue_to(:queue_name, MyWorker, 'foo', 1, :bat => 'bar')
167
+ # Sidekiq::Client.enqueue_to(:queue_name, MyJob, 'foo', 1, :bat => 'bar')
157
168
  #
158
169
  def enqueue_to(queue, klass, *args)
159
170
  klass.client_push("queue" => queue, "class" => klass, "args" => args)
160
171
  end
161
172
 
162
173
  # Example usage:
163
- # Sidekiq::Client.enqueue_to_in(:queue_name, 3.minutes, MyWorker, 'foo', 1, :bat => 'bar')
174
+ # Sidekiq::Client.enqueue_to_in(:queue_name, 3.minutes, MyJob, 'foo', 1, :bat => 'bar')
164
175
  #
165
176
  def enqueue_to_in(queue, interval, klass, *args)
166
177
  int = interval.to_f
@@ -174,7 +185,7 @@ module Sidekiq
174
185
  end
175
186
 
176
187
  # Example usage:
177
- # Sidekiq::Client.enqueue_in(3.minutes, MyWorker, 'foo', 1, :bat => 'bar')
188
+ # Sidekiq::Client.enqueue_in(3.minutes, MyJob, 'foo', 1, :bat => 'bar')
178
189
  #
179
190
  def enqueue_in(interval, klass, *args)
180
191
  klass.perform_in(interval, *args)
@@ -185,8 +196,23 @@ module Sidekiq
185
196
 
186
197
  def raw_push(payloads)
187
198
  @redis_pool.with do |conn|
188
- conn.multi do
189
- atomic_push(conn, payloads)
199
+ retryable = true
200
+ begin
201
+ conn.pipelined do |pipeline|
202
+ atomic_push(pipeline, payloads)
203
+ end
204
+ rescue Redis::BaseError => ex
205
+ # 2550 Failover can cause the server to become a replica, need
206
+ # to disconnect and reopen the socket to get back to the primary.
207
+ # 4495 Use the same logic if we have a "Not enough replicas" error from the primary
208
+ # 4985 Use the same logic when a blocking command is force-unblocked
209
+ # The retry logic is copied from sidekiq.rb
210
+ if retryable && ex.message =~ /READONLY|NOREPLICAS|UNBLOCKED/
211
+ conn.disconnect!
212
+ retryable = false
213
+ retry
214
+ end
215
+ raise
190
216
  end
191
217
  end
192
218
  true
@@ -209,50 +235,5 @@ module Sidekiq
209
235
  conn.lpush("queue:#{queue}", to_push)
210
236
  end
211
237
  end
212
-
213
- def process_single(worker_class, item)
214
- queue = item["queue"]
215
-
216
- middleware.invoke(worker_class, item, queue, @redis_pool) do
217
- item
218
- end
219
- end
220
-
221
- def normalize_item(item)
222
- # 6.0.0 push_bulk bug, #4321
223
- # TODO Remove after a while...
224
- item.delete("at") if item.key?("at") && item["at"].nil?
225
-
226
- raise(ArgumentError, "Job must be a Hash with 'class' and 'args' keys: { 'class' => SomeWorker, 'args' => ['bob', 1, :foo => 'bar'] }") unless item.is_a?(Hash) && item.key?("class") && item.key?("args")
227
- raise(ArgumentError, "Job args must be an Array") unless item["args"].is_a?(Array)
228
- raise(ArgumentError, "Job class must be either a Class or String representation of the class name") unless item["class"].is_a?(Class) || item["class"].is_a?(String)
229
- raise(ArgumentError, "Job 'at' must be a Numeric timestamp") if item.key?("at") && !item["at"].is_a?(Numeric)
230
- raise(ArgumentError, "Job tags must be an Array") if item["tags"] && !item["tags"].is_a?(Array)
231
- # raise(ArgumentError, "Arguments must be native JSON types, see https://github.com/mperham/sidekiq/wiki/Best-Practices") unless JSON.load(JSON.dump(item['args'])) == item['args']
232
-
233
- # merge in the default sidekiq_options for the item's class and/or wrapped element
234
- # this allows ActiveJobs to control sidekiq_options too.
235
- defaults = normalized_hash(item["class"])
236
- defaults = defaults.merge(item["wrapped"].get_sidekiq_options) if item["wrapped"].respond_to?("get_sidekiq_options")
237
- item = defaults.merge(item)
238
-
239
- raise(ArgumentError, "Job must include a valid queue name") if item["queue"].nil? || item["queue"] == ""
240
-
241
- item["class"] = item["class"].to_s
242
- item["queue"] = item["queue"].to_s
243
- item["jid"] ||= SecureRandom.hex(12)
244
- item["created_at"] ||= Time.now.to_f
245
-
246
- item
247
- end
248
-
249
- def normalized_hash(item_class)
250
- if item_class.is_a?(Class)
251
- raise(ArgumentError, "Message must include a Sidekiq::Worker class, not class name: #{item_class.ancestors.inspect}") unless item_class.respond_to?("get_sidekiq_options")
252
- item_class.get_sidekiq_options
253
- else
254
- Sidekiq.default_worker_options
255
- end
256
- end
257
238
  end
258
239
  end
data/lib/sidekiq/delay.rb CHANGED
@@ -3,6 +3,8 @@
3
3
  module Sidekiq
4
4
  module Extensions
5
5
  def self.enable_delay!
6
+ warn "Sidekiq's Delayed Extensions will be removed in Sidekiq 7.0", uplevel: 1
7
+
6
8
  if defined?(::ActiveSupport)
7
9
  require "sidekiq/extensions/active_record"
8
10
  require "sidekiq/extensions/action_mailer"
@@ -5,9 +5,10 @@ require "sidekiq/extensions/generic_proxy"
5
5
  module Sidekiq
6
6
  module Extensions
7
7
  ##
8
- # Adds 'delay', 'delay_for' and `delay_until` methods to ActionMailer to offload arbitrary email
9
- # delivery to Sidekiq. Example:
8
+ # Adds +delay+, +delay_for+ and +delay_until+ methods to ActionMailer to offload arbitrary email
9
+ # delivery to Sidekiq.
10
10
  #
11
+ # @example
11
12
  # UserMailer.delay.send_welcome_email(new_user)
12
13
  # UserMailer.delay_for(5.days).send_welcome_email(new_user)
13
14
  # UserMailer.delay_until(5.days.from_now).send_welcome_email(new_user)
@@ -5,10 +5,11 @@ require "sidekiq/extensions/generic_proxy"
5
5
  module Sidekiq
6
6
  module Extensions
7
7
  ##
8
- # Adds 'delay', 'delay_for' and `delay_until` methods to ActiveRecord to offload instance method
9
- # execution to Sidekiq. Examples:
8
+ # Adds +delay+, +delay_for+ and +delay_until+ methods to ActiveRecord to offload instance method
9
+ # execution to Sidekiq.
10
10
  #
11
- # User.recent_signups.each { |user| user.delay.mark_as_awesome }
11
+ # @example
12
+ # User.recent_signups.each { |user| user.delay.mark_as_awesome }
12
13
  #
13
14
  # Please note, this is not recommended as this will serialize the entire
14
15
  # object to Redis. Your Sidekiq jobs should pass IDs, not entire instances.
@@ -5,11 +5,12 @@ require "sidekiq/extensions/generic_proxy"
5
5
  module Sidekiq
6
6
  module Extensions
7
7
  ##
8
- # Adds 'delay', 'delay_for' and `delay_until` methods to all Classes to offload class method
9
- # execution to Sidekiq. Examples:
8
+ # Adds `delay`, `delay_for` and `delay_until` methods to all Classes to offload class method
9
+ # execution to Sidekiq.
10
10
  #
11
- # User.delay.delete_inactive
12
- # Wikipedia.delay.download_changes_for(Date.today)
11
+ # @example
12
+ # User.delay.delete_inactive
13
+ # Wikipedia.delay.download_changes_for(Date.today)
13
14
  #
14
15
  class DelayedClass
15
16
  include Sidekiq::Worker
@@ -10,7 +10,7 @@ module Sidekiq
10
10
  def initialize(performable, target, options = {})
11
11
  @performable = performable
12
12
  @target = target
13
- @opts = options
13
+ @opts = options.transform_keys(&:to_s)
14
14
  end
15
15
 
16
16
  def method_missing(name, *args)
@@ -24,7 +24,9 @@ module Sidekiq
24
24
  if marshalled.size > SIZE_LIMIT
25
25
  ::Sidekiq.logger.warn { "#{@target}.#{name} job argument is #{marshalled.bytesize} bytes, you should refactor it to reduce the size" }
26
26
  end
27
- @performable.client_push({"class" => @performable, "args" => [marshalled]}.merge(@opts))
27
+ @performable.client_push({"class" => @performable,
28
+ "args" => [marshalled],
29
+ "display_class" => "#{@target}.#{name}"}.merge(@opts))
28
30
  end
29
31
  end
30
32
  end
data/lib/sidekiq/fetch.rb CHANGED
@@ -25,8 +25,10 @@ module Sidekiq
25
25
  }
26
26
 
27
27
  def initialize(options)
28
- @strictly_ordered_queues = !!options[:strict]
29
- @queues = options[:queues].map { |q| "queue:#{q}" }
28
+ raise ArgumentError, "missing queue list" unless options[:queues]
29
+ @options = options
30
+ @strictly_ordered_queues = !!@options[:strict]
31
+ @queues = @options[:queues].map { |q| "queue:#{q}" }
30
32
  if @strictly_ordered_queues
31
33
  @queues.uniq!
32
34
  @queues << TIMEOUT
@@ -34,28 +36,19 @@ module Sidekiq
34
36
  end
35
37
 
36
38
  def retrieve_work
37
- work = Sidekiq.redis { |conn| conn.brpop(*queues_cmd) }
38
- UnitOfWork.new(*work) if work
39
- end
40
-
41
- # Creating the Redis#brpop command takes into account any
42
- # configured queue weights. By default Redis#brpop returns
43
- # data from the first queue that has pending elements. We
44
- # recreate the queue command each time we invoke Redis#brpop
45
- # to honor weights and avoid queue starvation.
46
- def queues_cmd
47
- if @strictly_ordered_queues
48
- @queues
49
- else
50
- queues = @queues.shuffle!.uniq
51
- queues << TIMEOUT
52
- queues
39
+ qs = queues_cmd
40
+ # 4825 Sidekiq Pro with all queues paused will return an
41
+ # empty set of queues with a trailing TIMEOUT value.
42
+ if qs.size <= 1
43
+ sleep(TIMEOUT)
44
+ return nil
53
45
  end
46
+
47
+ work = Sidekiq.redis { |conn| conn.brpop(*qs) }
48
+ UnitOfWork.new(*work) if work
54
49
  end
55
50
 
56
- # By leaving this as a class method, it can be pluggable and used by the Manager actor. Making it
57
- # an instance method will make it async to the Fetcher actor
58
- def self.bulk_requeue(inprogress, options)
51
+ def bulk_requeue(inprogress, options)
59
52
  return if inprogress.empty?
60
53
 
61
54
  Sidekiq.logger.debug { "Re-queueing terminated jobs" }
@@ -66,9 +59,9 @@ module Sidekiq
66
59
  end
67
60
 
68
61
  Sidekiq.redis do |conn|
69
- conn.pipelined do
62
+ conn.pipelined do |pipeline|
70
63
  jobs_to_requeue.each do |queue, jobs|
71
- conn.rpush(queue, jobs)
64
+ pipeline.rpush(queue, jobs)
72
65
  end
73
66
  end
74
67
  end
@@ -76,5 +69,21 @@ module Sidekiq
76
69
  rescue => ex
77
70
  Sidekiq.logger.warn("Failed to requeue #{inprogress.size} jobs: #{ex.message}")
78
71
  end
72
+
73
+ # Creating the Redis#brpop command takes into account any
74
+ # configured queue weights. By default Redis#brpop returns
75
+ # data from the first queue that has pending elements. We
76
+ # recreate the queue command each time we invoke Redis#brpop
77
+ # to honor weights and avoid queue starvation.
78
+ def queues_cmd
79
+ if @strictly_ordered_queues
80
+ @queues
81
+ else
82
+ permute = @queues.shuffle
83
+ permute.uniq!
84
+ permute << TIMEOUT
85
+ permute
86
+ end
87
+ end
79
88
  end
80
89
  end
@@ -0,0 +1,13 @@
1
+ require "sidekiq/worker"
2
+
3
+ module Sidekiq
4
+ # Sidekiq::Job is a new alias for Sidekiq::Worker as of Sidekiq 6.3.0.
5
+ # Use `include Sidekiq::Job` rather than `include Sidekiq::Worker`.
6
+ #
7
+ # The term "worker" is too generic and overly confusing, used in several
8
+ # different contexts meaning different things. Many people call a Sidekiq
9
+ # process a "worker". Some people call the thread that executes jobs a
10
+ # "worker". This change brings Sidekiq closer to ActiveJob where your job
11
+ # classes extend ApplicationJob.
12
+ Job = Worker
13
+ end
@@ -12,46 +12,34 @@ module Sidekiq
12
12
 
13
13
  yield
14
14
 
15
- with_elapsed_time_context(start) do
16
- @logger.info("done")
17
- end
15
+ Sidekiq::Context.add(:elapsed, elapsed(start))
16
+ @logger.info("done")
18
17
  rescue Exception
19
- with_elapsed_time_context(start) do
20
- @logger.info("fail")
21
- end
18
+ Sidekiq::Context.add(:elapsed, elapsed(start))
19
+ @logger.info("fail")
22
20
 
23
21
  raise
24
22
  end
25
23
 
26
24
  def prepare(job_hash, &block)
27
- level = job_hash["log_level"]
28
- if level
29
- @logger.log_at(level) do
30
- Sidekiq::Context.with(job_hash_context(job_hash), &block)
31
- end
32
- else
33
- Sidekiq::Context.with(job_hash_context(job_hash), &block)
34
- end
35
- end
36
-
37
- def job_hash_context(job_hash)
38
25
  # If we're using a wrapper class, like ActiveJob, use the "wrapped"
39
26
  # attribute to expose the underlying thing.
40
27
  h = {
41
- class: job_hash["wrapped"] || job_hash["class"],
28
+ class: job_hash["display_class"] || job_hash["wrapped"] || job_hash["class"],
42
29
  jid: job_hash["jid"]
43
30
  }
44
- h[:bid] = job_hash["bid"] if job_hash["bid"]
45
- h[:tags] = job_hash["tags"] if job_hash["tags"]
46
- h
47
- end
48
-
49
- def with_elapsed_time_context(start, &block)
50
- Sidekiq::Context.with(elapsed_time_context(start), &block)
51
- end
31
+ h[:bid] = job_hash["bid"] if job_hash.has_key?("bid")
32
+ h[:tags] = job_hash["tags"] if job_hash.has_key?("tags")
52
33
 
53
- def elapsed_time_context(start)
54
- {elapsed: elapsed(start).to_s}
34
+ Thread.current[:sidekiq_context] = h
35
+ level = job_hash["log_level"]
36
+ if level
37
+ @logger.log_at(level, &block)
38
+ else
39
+ yield
40
+ end
41
+ ensure
42
+ Thread.current[:sidekiq_context] = nil
55
43
  end
56
44
 
57
45
  private
@@ -25,18 +25,19 @@ module Sidekiq
25
25
  #
26
26
  # A job looks like:
27
27
  #
28
- # { 'class' => 'HardWorker', 'args' => [1, 2, 'foo'], 'retry' => true }
28
+ # { 'class' => 'HardJob', 'args' => [1, 2, 'foo'], 'retry' => true }
29
29
  #
30
30
  # The 'retry' option also accepts a number (in place of 'true'):
31
31
  #
32
- # { 'class' => 'HardWorker', 'args' => [1, 2, 'foo'], 'retry' => 5 }
32
+ # { 'class' => 'HardJob', 'args' => [1, 2, 'foo'], 'retry' => 5 }
33
33
  #
34
34
  # The job will be retried this number of times before giving up. (If simply
35
35
  # 'true', Sidekiq retries 25 times)
36
36
  #
37
- # We'll add a bit more data to the job to support retries:
37
+ # Relevant options for job retries:
38
38
  #
39
- # * 'queue' - the queue to use
39
+ # * 'queue' - the queue for the initial job
40
+ # * 'retry_queue' - if job retries should be pushed to a different (e.g. lower priority) queue
40
41
  # * 'retry_count' - number of times we've retried so far.
41
42
  # * 'error_message' - the message from the exception
42
43
  # * 'error_class' - the exception class
@@ -52,15 +53,17 @@ module Sidekiq
52
53
  #
53
54
  # Sidekiq.options[:max_retries] = 7
54
55
  #
55
- # or limit the number of retries for a particular worker with:
56
+ # or limit the number of retries for a particular job and send retries to
57
+ # a low priority queue with:
56
58
  #
57
- # class MyWorker
58
- # include Sidekiq::Worker
59
- # sidekiq_options :retry => 10
59
+ # class MyJob
60
+ # include Sidekiq::Job
61
+ # sidekiq_options retry: 10, retry_queue: 'low'
60
62
  # end
61
63
  #
62
64
  class JobRetry
63
65
  class Handled < ::RuntimeError; end
66
+
64
67
  class Skip < Handled; end
65
68
 
66
69
  include Sidekiq::Util
@@ -73,7 +76,7 @@ module Sidekiq
73
76
 
74
77
  # The global retry handler requires only the barest of data.
75
78
  # We want to be able to retry as much as possible so we don't
76
- # require the worker to be instantiated.
79
+ # require the job to be instantiated.
77
80
  def global(jobstr, queue)
78
81
  yield
79
82
  rescue Handled => ex
@@ -100,14 +103,14 @@ module Sidekiq
100
103
  end
101
104
 
102
105
  # The local retry support means that any errors that occur within
103
- # this block can be associated with the given worker instance.
106
+ # this block can be associated with the given job instance.
104
107
  # This is required to support the `sidekiq_retries_exhausted` block.
105
108
  #
106
109
  # Note that any exception from the block is wrapped in the Skip
107
110
  # exception so the global block does not reprocess the error. The
108
111
  # Skip exception is unwrapped within Sidekiq::Processor#process before
109
112
  # calling the handle_exception handlers.
110
- def local(worker, jobstr, queue)
113
+ def local(jobinst, jobstr, queue)
111
114
  yield
112
115
  rescue Handled => ex
113
116
  raise ex
@@ -120,11 +123,11 @@ module Sidekiq
120
123
 
121
124
  msg = Sidekiq.load_json(jobstr)
122
125
  if msg["retry"].nil?
123
- msg["retry"] = worker.class.get_sidekiq_options["retry"]
126
+ msg["retry"] = jobinst.class.get_sidekiq_options["retry"]
124
127
  end
125
128
 
126
129
  raise e unless msg["retry"]
127
- attempt_retry(worker, msg, queue, e)
130
+ attempt_retry(jobinst, msg, queue, e)
128
131
  # We've handled this error associated with this job, don't
129
132
  # need to handle it at the global level
130
133
  raise Skip
@@ -132,10 +135,10 @@ module Sidekiq
132
135
 
133
136
  private
134
137
 
135
- # Note that +worker+ can be nil here if an error is raised before we can
136
- # instantiate the worker instance. All access must be guarded and
138
+ # Note that +jobinst+ can be nil here if an error is raised before we can
139
+ # instantiate the job instance. All access must be guarded and
137
140
  # best effort.
138
- def attempt_retry(worker, msg, queue, exception)
141
+ def attempt_retry(jobinst, msg, queue, exception)
139
142
  max_retry_attempts = retry_attempts_from(msg["retry"], @max_retries)
140
143
 
141
144
  msg["queue"] = (msg["retry_queue"] || queue)
@@ -167,7 +170,7 @@ module Sidekiq
167
170
  end
168
171
 
169
172
  if count < max_retry_attempts
170
- delay = delay_for(worker, count, exception)
173
+ delay = delay_for(jobinst, count, exception)
171
174
  # Logging here can break retries if the logging device raises ENOSPC #3979
172
175
  # logger.debug { "Failure! Retry #{count} in #{delay} seconds" }
173
176
  retry_at = Time.now.to_f + delay
@@ -177,13 +180,13 @@ module Sidekiq
177
180
  end
178
181
  else
179
182
  # Goodbye dear message, you (re)tried your best I'm sure.
180
- retries_exhausted(worker, msg, exception)
183
+ retries_exhausted(jobinst, msg, exception)
181
184
  end
182
185
  end
183
186
 
184
- def retries_exhausted(worker, msg, exception)
187
+ def retries_exhausted(jobinst, msg, exception)
185
188
  begin
186
- block = worker&.sidekiq_retries_exhausted_block
189
+ block = jobinst&.sidekiq_retries_exhausted_block
187
190
  block&.call(msg, exception)
188
191
  rescue => e
189
192
  handle_exception(e, {context: "Error calling retries_exhausted", job: msg})
@@ -212,23 +215,19 @@ module Sidekiq
212
215
  end
213
216
  end
214
217
 
215
- def delay_for(worker, count, exception)
216
- if worker&.sidekiq_retry_in_block
217
- custom_retry_in = retry_in(worker, count, exception).to_i
218
- return custom_retry_in if custom_retry_in > 0
218
+ def delay_for(jobinst, count, exception)
219
+ jitter = rand(10) * (count + 1)
220
+ if jobinst&.sidekiq_retry_in_block
221
+ custom_retry_in = retry_in(jobinst, count, exception).to_i
222
+ return custom_retry_in + jitter if custom_retry_in > 0
219
223
  end
220
- seconds_to_delay(count)
221
- end
222
-
223
- # delayed_job uses the same basic formula
224
- def seconds_to_delay(count)
225
- (count**4) + 15 + (rand(30) * (count + 1))
224
+ (count**4) + 15 + jitter
226
225
  end
227
226
 
228
- def retry_in(worker, count, exception)
229
- worker.sidekiq_retry_in_block.call(count, exception)
227
+ def retry_in(jobinst, count, exception)
228
+ jobinst.sidekiq_retry_in_block.call(count, exception)
230
229
  rescue Exception => e
231
- handle_exception(e, {context: "Failure scheduling retry using the defined `sidekiq_retry_in` in #{worker.class.name}, falling back to default"})
230
+ handle_exception(e, {context: "Failure scheduling retry using the defined `sidekiq_retry_in` in #{jobinst.class.name}, falling back to default"})
232
231
  nil
233
232
  end
234
233