sidekiq 5.2.9 → 6.2.1

Sign up to get free protection for your applications and to get access to all the features.

Potentially problematic release.


This version of sidekiq might be problematic. Click here for more details.

Files changed (92) hide show
  1. checksums.yaml +4 -4
  2. data/Changes.md +232 -0
  3. data/README.md +18 -34
  4. data/bin/sidekiq +26 -2
  5. data/bin/sidekiqload +32 -24
  6. data/bin/sidekiqmon +8 -0
  7. data/lib/generators/sidekiq/templates/worker_test.rb.erb +1 -1
  8. data/lib/generators/sidekiq/worker_generator.rb +21 -13
  9. data/lib/sidekiq/api.rb +257 -219
  10. data/lib/sidekiq/cli.rb +144 -180
  11. data/lib/sidekiq/client.rb +64 -48
  12. data/lib/sidekiq/delay.rb +5 -6
  13. data/lib/sidekiq/exception_handler.rb +10 -12
  14. data/lib/sidekiq/extensions/action_mailer.rb +13 -22
  15. data/lib/sidekiq/extensions/active_record.rb +13 -10
  16. data/lib/sidekiq/extensions/class_methods.rb +14 -11
  17. data/lib/sidekiq/extensions/generic_proxy.rb +4 -4
  18. data/lib/sidekiq/fetch.rb +38 -31
  19. data/lib/sidekiq/job_logger.rb +45 -7
  20. data/lib/sidekiq/job_retry.rb +62 -61
  21. data/lib/sidekiq/launcher.rb +142 -52
  22. data/lib/sidekiq/logger.rb +166 -0
  23. data/lib/sidekiq/manager.rb +11 -13
  24. data/lib/sidekiq/middleware/chain.rb +15 -5
  25. data/lib/sidekiq/middleware/i18n.rb +5 -7
  26. data/lib/sidekiq/monitor.rb +133 -0
  27. data/lib/sidekiq/paginator.rb +18 -14
  28. data/lib/sidekiq/processor.rb +71 -70
  29. data/lib/sidekiq/rails.rb +29 -37
  30. data/lib/sidekiq/redis_connection.rb +50 -48
  31. data/lib/sidekiq/scheduled.rb +28 -29
  32. data/lib/sidekiq/sd_notify.rb +149 -0
  33. data/lib/sidekiq/systemd.rb +24 -0
  34. data/lib/sidekiq/testing/inline.rb +2 -1
  35. data/lib/sidekiq/testing.rb +35 -24
  36. data/lib/sidekiq/util.rb +45 -16
  37. data/lib/sidekiq/version.rb +2 -1
  38. data/lib/sidekiq/web/action.rb +15 -11
  39. data/lib/sidekiq/web/application.rb +84 -74
  40. data/lib/sidekiq/web/csrf_protection.rb +180 -0
  41. data/lib/sidekiq/web/helpers.rb +108 -79
  42. data/lib/sidekiq/web/router.rb +23 -19
  43. data/lib/sidekiq/web.rb +60 -105
  44. data/lib/sidekiq/worker.rb +126 -102
  45. data/lib/sidekiq.rb +69 -44
  46. data/sidekiq.gemspec +23 -16
  47. data/web/assets/images/apple-touch-icon.png +0 -0
  48. data/web/assets/javascripts/application.js +25 -27
  49. data/web/assets/javascripts/dashboard.js +4 -23
  50. data/web/assets/stylesheets/application-dark.css +160 -0
  51. data/web/assets/stylesheets/application.css +33 -8
  52. data/web/locales/de.yml +14 -2
  53. data/web/locales/en.yml +2 -0
  54. data/web/locales/fr.yml +3 -3
  55. data/web/locales/ja.yml +4 -1
  56. data/web/locales/lt.yml +83 -0
  57. data/web/locales/pl.yml +4 -4
  58. data/web/locales/ru.yml +4 -0
  59. data/web/locales/vi.yml +83 -0
  60. data/web/views/_job_info.erb +2 -1
  61. data/web/views/busy.erb +51 -17
  62. data/web/views/dead.erb +2 -2
  63. data/web/views/layout.erb +2 -0
  64. data/web/views/morgue.erb +5 -2
  65. data/web/views/queue.erb +11 -2
  66. data/web/views/queues.erb +9 -1
  67. data/web/views/retries.erb +5 -2
  68. data/web/views/retry.erb +2 -2
  69. data/web/views/scheduled.erb +5 -2
  70. metadata +27 -60
  71. data/.circleci/config.yml +0 -61
  72. data/.github/contributing.md +0 -32
  73. data/.github/issue_template.md +0 -11
  74. data/.gitignore +0 -15
  75. data/.travis.yml +0 -11
  76. data/3.0-Upgrade.md +0 -70
  77. data/4.0-Upgrade.md +0 -53
  78. data/5.0-Upgrade.md +0 -56
  79. data/COMM-LICENSE +0 -97
  80. data/Ent-Changes.md +0 -238
  81. data/Gemfile +0 -23
  82. data/Pro-2.0-Upgrade.md +0 -138
  83. data/Pro-3.0-Upgrade.md +0 -44
  84. data/Pro-4.0-Upgrade.md +0 -35
  85. data/Pro-Changes.md +0 -759
  86. data/Rakefile +0 -9
  87. data/bin/sidekiqctl +0 -20
  88. data/code_of_conduct.md +0 -50
  89. data/lib/sidekiq/core_ext.rb +0 -1
  90. data/lib/sidekiq/ctl.rb +0 -221
  91. data/lib/sidekiq/logging.rb +0 -122
  92. data/lib/sidekiq/middleware/server/active_record.rb +0 -23
@@ -1,10 +1,10 @@
1
1
  # frozen_string_literal: true
2
- require 'securerandom'
3
- require 'sidekiq/middleware/chain'
2
+
3
+ require "securerandom"
4
+ require "sidekiq/middleware/chain"
4
5
 
5
6
  module Sidekiq
6
7
  class Client
7
-
8
8
  ##
9
9
  # Define client-side middleware:
10
10
  #
@@ -19,7 +19,7 @@ module Sidekiq
19
19
  #
20
20
  def middleware(&block)
21
21
  @chain ||= Sidekiq.client_middleware
22
- if block_given?
22
+ if block
23
23
  @chain = @chain.dup
24
24
  yield @chain
25
25
  end
@@ -38,7 +38,7 @@ module Sidekiq
38
38
  # Generally this is only needed for very large Sidekiq installs processing
39
39
  # thousands of jobs per second. I don't recommend sharding unless you
40
40
  # cannot scale any other way (e.g. splitting your app into smaller apps).
41
- def initialize(redis_pool=nil)
41
+ def initialize(redis_pool = nil)
42
42
  @redis_pool = redis_pool || Thread.current[:sidekiq_via_pool] || Sidekiq.redis_pool
43
43
  end
44
44
 
@@ -68,11 +68,11 @@ module Sidekiq
68
68
  #
69
69
  def push(item)
70
70
  normed = normalize_item(item)
71
- payload = process_single(item['class'], normed)
71
+ payload = process_single(item["class"], normed)
72
72
 
73
73
  if payload
74
74
  raw_push([payload])
75
- payload['jid']
75
+ payload["jid"]
76
76
  end
77
77
  end
78
78
 
@@ -90,19 +90,25 @@ module Sidekiq
90
90
  # Returns an array of the of pushed jobs' jids. The number of jobs pushed can be less
91
91
  # than the number given if the middleware stopped processing for one or more jobs.
92
92
  def push_bulk(items)
93
- arg = items['args'].first
94
- return [] unless arg # no jobs to push
95
- raise ArgumentError, "Bulk arguments must be an Array of Arrays: [[1], [2]]" if !arg.is_a?(Array)
93
+ args = items["args"]
94
+ raise ArgumentError, "Bulk arguments must be an Array of Arrays: [[1], [2]]" unless args.is_a?(Array) && args.all?(Array)
95
+ return [] if args.empty? # no jobs to push
96
+
97
+ at = items.delete("at")
98
+ raise ArgumentError, "Job 'at' must be a Numeric or an Array of Numeric timestamps" if at && (Array(at).empty? || !Array(at).all?(Numeric))
99
+ raise ArgumentError, "Job 'at' Array must have same size as 'args' Array" if at.is_a?(Array) && at.size != args.size
96
100
 
97
101
  normed = normalize_item(items)
98
- payloads = items['args'].map do |args|
99
- copy = normed.merge('args' => args, 'jid' => SecureRandom.hex(12), 'enqueued_at' => Time.now.to_f)
100
- result = process_single(items['class'], copy)
101
- result ? result : nil
102
- end.compact
103
-
104
- raw_push(payloads) if !payloads.empty?
105
- payloads.collect { |payload| payload['jid'] }
102
+ payloads = args.map.with_index { |job_args, index|
103
+ copy = normed.merge("args" => job_args, "jid" => SecureRandom.hex(12), "enqueued_at" => Time.now.to_f)
104
+ copy["at"] = (at.is_a?(Array) ? at[index] : at) if at
105
+
106
+ result = process_single(items["class"], copy)
107
+ result || nil
108
+ }.compact
109
+
110
+ raw_push(payloads) unless payloads.empty?
111
+ payloads.collect { |payload| payload["jid"] }
106
112
  end
107
113
 
108
114
  # Allows sharding of jobs across any number of Redis instances. All jobs
@@ -127,7 +133,6 @@ module Sidekiq
127
133
  end
128
134
 
129
135
  class << self
130
-
131
136
  def push(item)
132
137
  new.push(item)
133
138
  end
@@ -145,14 +150,14 @@ module Sidekiq
145
150
  # Messages are enqueued to the 'default' queue.
146
151
  #
147
152
  def enqueue(klass, *args)
148
- klass.client_push('class' => klass, 'args' => args)
153
+ klass.client_push("class" => klass, "args" => args)
149
154
  end
150
155
 
151
156
  # Example usage:
152
157
  # Sidekiq::Client.enqueue_to(:queue_name, MyWorker, 'foo', 1, :bat => 'bar')
153
158
  #
154
159
  def enqueue_to(queue, klass, *args)
155
- klass.client_push('queue' => queue, 'class' => klass, 'args' => args)
160
+ klass.client_push("queue" => queue, "class" => klass, "args" => args)
156
161
  end
157
162
 
158
163
  # Example usage:
@@ -163,8 +168,8 @@ module Sidekiq
163
168
  now = Time.now.to_f
164
169
  ts = (int < 1_000_000_000 ? now + int : int)
165
170
 
166
- item = { 'class' => klass, 'args' => args, 'at' => ts, 'queue' => queue }
167
- item.delete('at') if ts <= now
171
+ item = {"class" => klass, "args" => args, "at" => ts, "queue" => queue}
172
+ item.delete("at") if ts <= now
168
173
 
169
174
  klass.client_push(item)
170
175
  end
@@ -189,51 +194,62 @@ module Sidekiq
189
194
  end
190
195
 
191
196
  def atomic_push(conn, payloads)
192
- if payloads.first['at']
193
- conn.zadd('schedule', payloads.map do |hash|
194
- at = hash.delete('at').to_s
197
+ if payloads.first.key?("at")
198
+ conn.zadd("schedule", payloads.map { |hash|
199
+ at = hash.delete("at").to_s
195
200
  [at, Sidekiq.dump_json(hash)]
196
- end)
201
+ })
197
202
  else
198
- q = payloads.first['queue']
203
+ queue = payloads.first["queue"]
199
204
  now = Time.now.to_f
200
- to_push = payloads.map do |entry|
201
- entry['enqueued_at'] = now
205
+ to_push = payloads.map { |entry|
206
+ entry["enqueued_at"] = now
202
207
  Sidekiq.dump_json(entry)
203
- end
204
- conn.sadd('queues', q)
205
- conn.lpush("queue:#{q}", to_push)
208
+ }
209
+ conn.sadd("queues", queue)
210
+ conn.lpush("queue:#{queue}", to_push)
206
211
  end
207
212
  end
208
213
 
209
214
  def process_single(worker_class, item)
210
- queue = item['queue']
215
+ queue = item["queue"]
211
216
 
212
217
  middleware.invoke(worker_class, item, queue, @redis_pool) do
213
218
  item
214
219
  end
215
220
  end
216
221
 
222
+ def validate(item)
223
+ raise(ArgumentError, "Job must be a Hash with 'class' and 'args' keys: `#{item}`") unless item.is_a?(Hash) && item.key?("class") && item.key?("args")
224
+ raise(ArgumentError, "Job args must be an Array: `#{item}`") unless item["args"].is_a?(Array)
225
+ raise(ArgumentError, "Job class must be either a Class or String representation of the class name: `#{item}`") unless item["class"].is_a?(Class) || item["class"].is_a?(String)
226
+ raise(ArgumentError, "Job 'at' must be a Numeric timestamp: `#{item}`") if item.key?("at") && !item["at"].is_a?(Numeric)
227
+ raise(ArgumentError, "Job tags must be an Array: `#{item}`") if item["tags"] && !item["tags"].is_a?(Array)
228
+ end
229
+
217
230
  def normalize_item(item)
218
- raise(ArgumentError, "Job must be a Hash with 'class' and 'args' keys: { 'class' => SomeWorker, 'args' => ['bob', 1, :foo => 'bar'] }") unless item.is_a?(Hash) && item.has_key?('class') && item.has_key?('args')
219
- raise(ArgumentError, "Job args must be an Array") unless item['args'].is_a?(Array)
220
- raise(ArgumentError, "Job class must be either a Class or String representation of the class name") unless item['class'].is_a?(Class) || item['class'].is_a?(String)
221
- raise(ArgumentError, "Job 'at' must be a Numeric timestamp") if item.has_key?('at') && !item['at'].is_a?(Numeric)
222
- #raise(ArgumentError, "Arguments must be native JSON types, see https://github.com/mperham/sidekiq/wiki/Best-Practices") unless JSON.load(JSON.dump(item['args'])) == item['args']
223
-
224
- normalized_hash(item['class'])
225
- .each{ |key, value| item[key] = value if item[key].nil? }
226
-
227
- item['class'] = item['class'].to_s
228
- item['queue'] = item['queue'].to_s
229
- item['jid'] ||= SecureRandom.hex(12)
230
- item['created_at'] ||= Time.now.to_f
231
+ validate(item)
232
+ # raise(ArgumentError, "Arguments must be native JSON types, see https://github.com/mperham/sidekiq/wiki/Best-Practices") unless JSON.load(JSON.dump(item['args'])) == item['args']
233
+
234
+ # merge in the default sidekiq_options for the item's class and/or wrapped element
235
+ # this allows ActiveJobs to control sidekiq_options too.
236
+ defaults = normalized_hash(item["class"])
237
+ defaults = defaults.merge(item["wrapped"].get_sidekiq_options) if item["wrapped"].respond_to?("get_sidekiq_options")
238
+ item = defaults.merge(item)
239
+
240
+ raise(ArgumentError, "Job must include a valid queue name") if item["queue"].nil? || item["queue"] == ""
241
+
242
+ item["class"] = item["class"].to_s
243
+ item["queue"] = item["queue"].to_s
244
+ item["jid"] ||= SecureRandom.hex(12)
245
+ item["created_at"] ||= Time.now.to_f
246
+
231
247
  item
232
248
  end
233
249
 
234
250
  def normalized_hash(item_class)
235
251
  if item_class.is_a?(Class)
236
- raise(ArgumentError, "Message must include a Sidekiq::Worker class, not class name: #{item_class.ancestors.inspect}") if !item_class.respond_to?('get_sidekiq_options')
252
+ raise(ArgumentError, "Message must include a Sidekiq::Worker class, not class name: #{item_class.ancestors.inspect}") unless item_class.respond_to?("get_sidekiq_options")
237
253
  item_class.get_sidekiq_options
238
254
  else
239
255
  Sidekiq.default_worker_options
data/lib/sidekiq/delay.rb CHANGED
@@ -1,11 +1,11 @@
1
1
  # frozen_string_literal: true
2
+
2
3
  module Sidekiq
3
4
  module Extensions
4
-
5
5
  def self.enable_delay!
6
6
  if defined?(::ActiveSupport)
7
- require 'sidekiq/extensions/active_record'
8
- require 'sidekiq/extensions/action_mailer'
7
+ require "sidekiq/extensions/active_record"
8
+ require "sidekiq/extensions/action_mailer"
9
9
 
10
10
  # Need to patch Psych so it can autoload classes whose names are serialized
11
11
  # in the delayed YAML.
@@ -19,7 +19,7 @@ module Sidekiq
19
19
  end
20
20
  end
21
21
 
22
- require 'sidekiq/extensions/class_methods'
22
+ require "sidekiq/extensions/class_methods"
23
23
  Module.__send__(:include, Sidekiq::Extensions::Klass)
24
24
  end
25
25
 
@@ -27,7 +27,7 @@ module Sidekiq
27
27
  def resolve_class(klass_name)
28
28
  return nil if !klass_name || klass_name.empty?
29
29
  # constantize
30
- names = klass_name.split('::')
30
+ names = klass_name.split("::")
31
31
  names.shift if names.empty? || names.first.empty?
32
32
 
33
33
  names.inject(Object) do |constant, name|
@@ -39,4 +39,3 @@ module Sidekiq
39
39
  end
40
40
  end
41
41
  end
42
-
@@ -1,12 +1,12 @@
1
1
  # frozen_string_literal: true
2
- require 'sidekiq'
2
+
3
+ require "sidekiq"
3
4
 
4
5
  module Sidekiq
5
6
  module ExceptionHandler
6
-
7
7
  class Logger
8
- def call(ex, ctxHash)
9
- Sidekiq.logger.warn(Sidekiq.dump_json(ctxHash)) if !ctxHash.empty?
8
+ def call(ex, ctx)
9
+ Sidekiq.logger.warn(Sidekiq.dump_json(ctx)) unless ctx.empty?
10
10
  Sidekiq.logger.warn("#{ex.class.name}: #{ex.message}")
11
11
  Sidekiq.logger.warn(ex.backtrace.join("\n")) unless ex.backtrace.nil?
12
12
  end
@@ -14,15 +14,13 @@ module Sidekiq
14
14
  Sidekiq.error_handlers << Sidekiq::ExceptionHandler::Logger.new
15
15
  end
16
16
 
17
- def handle_exception(ex, ctxHash={})
17
+ def handle_exception(ex, ctx = {})
18
18
  Sidekiq.error_handlers.each do |handler|
19
- begin
20
- handler.call(ex, ctxHash)
21
- rescue => ex
22
- Sidekiq.logger.error "!!! ERROR HANDLER THREW AN ERROR !!!"
23
- Sidekiq.logger.error ex
24
- Sidekiq.logger.error ex.backtrace.join("\n") unless ex.backtrace.nil?
25
- end
19
+ handler.call(ex, ctx)
20
+ rescue => ex
21
+ Sidekiq.logger.error "!!! ERROR HANDLER THREW AN ERROR !!!"
22
+ Sidekiq.logger.error ex
23
+ Sidekiq.logger.error ex.backtrace.join("\n") unless ex.backtrace.nil?
26
24
  end
27
25
  end
28
26
  end
@@ -1,12 +1,14 @@
1
1
  # frozen_string_literal: true
2
- require 'sidekiq/extensions/generic_proxy'
2
+
3
+ require "sidekiq/extensions/generic_proxy"
3
4
 
4
5
  module Sidekiq
5
6
  module Extensions
6
7
  ##
7
- # Adds 'delay', 'delay_for' and `delay_until` methods to ActionMailer to offload arbitrary email
8
- # delivery to Sidekiq. Example:
8
+ # Adds +delay+, +delay_for+ and +delay_until+ methods to ActionMailer to offload arbitrary email
9
+ # delivery to Sidekiq.
9
10
  #
11
+ # @example
10
12
  # UserMailer.delay.send_welcome_email(new_user)
11
13
  # UserMailer.delay_for(5.days).send_welcome_email(new_user)
12
14
  # UserMailer.delay_until(5.days.from_now).send_welcome_email(new_user)
@@ -19,39 +21,28 @@ module Sidekiq
19
21
  # The email method can return nil, which causes ActionMailer to return
20
22
  # an undeliverable empty message.
21
23
  if msg
22
- deliver(msg)
23
- else
24
- raise "#{target.name}##{method_name} returned an undeliverable mail object"
25
- end
26
- end
27
-
28
- private
29
-
30
- def deliver(msg)
31
- if msg.respond_to?(:deliver_now)
32
- # Rails 4.2/5.0
33
24
  msg.deliver_now
34
25
  else
35
- # Rails 3.2/4.0/4.1
36
- msg.deliver
26
+ raise "#{target.name}##{method_name} returned an undeliverable mail object"
37
27
  end
38
28
  end
39
29
  end
40
30
 
41
31
  module ActionMailer
42
- def sidekiq_delay(options={})
32
+ def sidekiq_delay(options = {})
43
33
  Proxy.new(DelayedMailer, self, options)
44
34
  end
45
- def sidekiq_delay_for(interval, options={})
46
- Proxy.new(DelayedMailer, self, options.merge('at' => Time.now.to_f + interval.to_f))
35
+
36
+ def sidekiq_delay_for(interval, options = {})
37
+ Proxy.new(DelayedMailer, self, options.merge("at" => Time.now.to_f + interval.to_f))
47
38
  end
48
- def sidekiq_delay_until(timestamp, options={})
49
- Proxy.new(DelayedMailer, self, options.merge('at' => timestamp.to_f))
39
+
40
+ def sidekiq_delay_until(timestamp, options = {})
41
+ Proxy.new(DelayedMailer, self, options.merge("at" => timestamp.to_f))
50
42
  end
51
43
  alias_method :delay, :sidekiq_delay
52
44
  alias_method :delay_for, :sidekiq_delay_for
53
45
  alias_method :delay_until, :sidekiq_delay_until
54
46
  end
55
-
56
47
  end
57
48
  end
@@ -1,13 +1,15 @@
1
1
  # frozen_string_literal: true
2
- require 'sidekiq/extensions/generic_proxy'
2
+
3
+ require "sidekiq/extensions/generic_proxy"
3
4
 
4
5
  module Sidekiq
5
6
  module Extensions
6
7
  ##
7
- # Adds 'delay', 'delay_for' and `delay_until` methods to ActiveRecord to offload instance method
8
- # execution to Sidekiq. Examples:
8
+ # Adds +delay+, +delay_for+ and +delay_until+ methods to ActiveRecord to offload instance method
9
+ # execution to Sidekiq.
9
10
  #
10
- # User.recent_signups.each { |user| user.delay.mark_as_awesome }
11
+ # @example
12
+ # User.recent_signups.each { |user| user.delay.mark_as_awesome }
11
13
  #
12
14
  # Please note, this is not recommended as this will serialize the entire
13
15
  # object to Redis. Your Sidekiq jobs should pass IDs, not entire instances.
@@ -22,19 +24,20 @@ module Sidekiq
22
24
  end
23
25
 
24
26
  module ActiveRecord
25
- def sidekiq_delay(options={})
27
+ def sidekiq_delay(options = {})
26
28
  Proxy.new(DelayedModel, self, options)
27
29
  end
28
- def sidekiq_delay_for(interval, options={})
29
- Proxy.new(DelayedModel, self, options.merge('at' => Time.now.to_f + interval.to_f))
30
+
31
+ def sidekiq_delay_for(interval, options = {})
32
+ Proxy.new(DelayedModel, self, options.merge("at" => Time.now.to_f + interval.to_f))
30
33
  end
31
- def sidekiq_delay_until(timestamp, options={})
32
- Proxy.new(DelayedModel, self, options.merge('at' => timestamp.to_f))
34
+
35
+ def sidekiq_delay_until(timestamp, options = {})
36
+ Proxy.new(DelayedModel, self, options.merge("at" => timestamp.to_f))
33
37
  end
34
38
  alias_method :delay, :sidekiq_delay
35
39
  alias_method :delay_for, :sidekiq_delay_for
36
40
  alias_method :delay_until, :sidekiq_delay_until
37
41
  end
38
-
39
42
  end
40
43
  end
@@ -1,14 +1,16 @@
1
1
  # frozen_string_literal: true
2
- require 'sidekiq/extensions/generic_proxy'
2
+
3
+ require "sidekiq/extensions/generic_proxy"
3
4
 
4
5
  module Sidekiq
5
6
  module Extensions
6
7
  ##
7
- # Adds 'delay', 'delay_for' and `delay_until` methods to all Classes to offload class method
8
- # execution to Sidekiq. Examples:
8
+ # Adds `delay`, `delay_for` and `delay_until` methods to all Classes to offload class method
9
+ # execution to Sidekiq.
9
10
  #
10
- # User.delay.delete_inactive
11
- # Wikipedia.delay.download_changes_for(Date.today)
11
+ # @example
12
+ # User.delay.delete_inactive
13
+ # Wikipedia.delay.download_changes_for(Date.today)
12
14
  #
13
15
  class DelayedClass
14
16
  include Sidekiq::Worker
@@ -20,20 +22,21 @@ module Sidekiq
20
22
  end
21
23
 
22
24
  module Klass
23
- def sidekiq_delay(options={})
25
+ def sidekiq_delay(options = {})
24
26
  Proxy.new(DelayedClass, self, options)
25
27
  end
26
- def sidekiq_delay_for(interval, options={})
27
- Proxy.new(DelayedClass, self, options.merge('at' => Time.now.to_f + interval.to_f))
28
+
29
+ def sidekiq_delay_for(interval, options = {})
30
+ Proxy.new(DelayedClass, self, options.merge("at" => Time.now.to_f + interval.to_f))
28
31
  end
29
- def sidekiq_delay_until(timestamp, options={})
30
- Proxy.new(DelayedClass, self, options.merge('at' => timestamp.to_f))
32
+
33
+ def sidekiq_delay_until(timestamp, options = {})
34
+ Proxy.new(DelayedClass, self, options.merge("at" => timestamp.to_f))
31
35
  end
32
36
  alias_method :delay, :sidekiq_delay
33
37
  alias_method :delay_for, :sidekiq_delay_for
34
38
  alias_method :delay_until, :sidekiq_delay_until
35
39
  end
36
-
37
40
  end
38
41
  end
39
42
 
@@ -1,12 +1,13 @@
1
1
  # frozen_string_literal: true
2
- require 'yaml'
2
+
3
+ require "yaml"
3
4
 
4
5
  module Sidekiq
5
6
  module Extensions
6
7
  SIZE_LIMIT = 8_192
7
8
 
8
9
  class Proxy < BasicObject
9
- def initialize(performable, target, options={})
10
+ def initialize(performable, target, options = {})
10
11
  @performable = performable
11
12
  @target = target
12
13
  @opts = options
@@ -23,9 +24,8 @@ module Sidekiq
23
24
  if marshalled.size > SIZE_LIMIT
24
25
  ::Sidekiq.logger.warn { "#{@target}.#{name} job argument is #{marshalled.bytesize} bytes, you should refactor it to reduce the size" }
25
26
  end
26
- @performable.client_push({ 'class' => @performable, 'args' => [marshalled] }.merge(@opts))
27
+ @performable.client_push({"class" => @performable, "args" => [marshalled]}.merge(@opts))
27
28
  end
28
29
  end
29
-
30
30
  end
31
31
  end
data/lib/sidekiq/fetch.rb CHANGED
@@ -1,5 +1,6 @@
1
1
  # frozen_string_literal: true
2
- require 'sidekiq'
2
+
3
+ require "sidekiq"
3
4
 
4
5
  module Sidekiq
5
6
  class BasicFetch
@@ -7,68 +8,60 @@ module Sidekiq
7
8
  # can check if the process is shutting down.
8
9
  TIMEOUT = 2
9
10
 
10
- UnitOfWork = Struct.new(:queue, :job) do
11
+ UnitOfWork = Struct.new(:queue, :job) {
11
12
  def acknowledge
12
13
  # nothing to do
13
14
  end
14
15
 
15
16
  def queue_name
16
- queue.sub(/.*queue:/, '')
17
+ queue.delete_prefix("queue:")
17
18
  end
18
19
 
19
20
  def requeue
20
21
  Sidekiq.redis do |conn|
21
- conn.rpush("queue:#{queue_name}", job)
22
+ conn.rpush(queue, job)
22
23
  end
23
24
  end
24
- end
25
+ }
25
26
 
26
27
  def initialize(options)
27
- @strictly_ordered_queues = !!options[:strict]
28
- @queues = options[:queues].map { |q| "queue:#{q}" }
28
+ raise ArgumentError, "missing queue list" unless options[:queues]
29
+ @options = options
30
+ @strictly_ordered_queues = !!@options[:strict]
31
+ @queues = @options[:queues].map { |q| "queue:#{q}" }
29
32
  if @strictly_ordered_queues
30
- @queues = @queues.uniq
33
+ @queues.uniq!
31
34
  @queues << TIMEOUT
32
35
  end
33
36
  end
34
37
 
35
38
  def retrieve_work
36
- work = Sidekiq.redis { |conn| conn.brpop(*queues_cmd) }
37
- UnitOfWork.new(*work) if work
38
- end
39
-
40
- # Creating the Redis#brpop command takes into account any
41
- # configured queue weights. By default Redis#brpop returns
42
- # data from the first queue that has pending elements. We
43
- # recreate the queue command each time we invoke Redis#brpop
44
- # to honor weights and avoid queue starvation.
45
- def queues_cmd
46
- if @strictly_ordered_queues
47
- @queues
48
- else
49
- queues = @queues.shuffle.uniq
50
- queues << TIMEOUT
51
- queues
39
+ qs = queues_cmd
40
+ # 4825 Sidekiq Pro with all queues paused will return an
41
+ # empty set of queues with a trailing TIMEOUT value.
42
+ if qs.size <= 1
43
+ sleep(2)
44
+ return nil
52
45
  end
53
- end
54
46
 
47
+ work = Sidekiq.redis { |conn| conn.brpop(*qs) }
48
+ UnitOfWork.new(*work) if work
49
+ end
55
50
 
56
- # By leaving this as a class method, it can be pluggable and used by the Manager actor. Making it
57
- # an instance method will make it async to the Fetcher actor
58
- def self.bulk_requeue(inprogress, options)
51
+ def bulk_requeue(inprogress, options)
59
52
  return if inprogress.empty?
60
53
 
61
54
  Sidekiq.logger.debug { "Re-queueing terminated jobs" }
62
55
  jobs_to_requeue = {}
63
56
  inprogress.each do |unit_of_work|
64
- jobs_to_requeue[unit_of_work.queue_name] ||= []
65
- jobs_to_requeue[unit_of_work.queue_name] << unit_of_work.job
57
+ jobs_to_requeue[unit_of_work.queue] ||= []
58
+ jobs_to_requeue[unit_of_work.queue] << unit_of_work.job
66
59
  end
67
60
 
68
61
  Sidekiq.redis do |conn|
69
62
  conn.pipelined do
70
63
  jobs_to_requeue.each do |queue, jobs|
71
- conn.rpush("queue:#{queue}", jobs)
64
+ conn.rpush(queue, jobs)
72
65
  end
73
66
  end
74
67
  end
@@ -77,5 +70,19 @@ module Sidekiq
77
70
  Sidekiq.logger.warn("Failed to requeue #{inprogress.size} jobs: #{ex.message}")
78
71
  end
79
72
 
73
+ # Creating the Redis#brpop command takes into account any
74
+ # configured queue weights. By default Redis#brpop returns
75
+ # data from the first queue that has pending elements. We
76
+ # recreate the queue command each time we invoke Redis#brpop
77
+ # to honor weights and avoid queue starvation.
78
+ def queues_cmd
79
+ if @strictly_ordered_queues
80
+ @queues
81
+ else
82
+ queues = @queues.shuffle!.uniq
83
+ queues << TIMEOUT
84
+ queues
85
+ end
86
+ end
80
87
  end
81
88
  end
@@ -1,25 +1,63 @@
1
1
  # frozen_string_literal: true
2
+
2
3
  module Sidekiq
3
4
  class JobLogger
5
+ def initialize(logger = Sidekiq.logger)
6
+ @logger = logger
7
+ end
4
8
 
5
9
  def call(item, queue)
6
10
  start = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
7
- logger.info("start")
11
+ @logger.info("start")
12
+
8
13
  yield
9
- logger.info("done: #{elapsed(start)} sec")
14
+
15
+ with_elapsed_time_context(start) do
16
+ @logger.info("done")
17
+ end
10
18
  rescue Exception
11
- logger.info("fail: #{elapsed(start)} sec")
19
+ with_elapsed_time_context(start) do
20
+ @logger.info("fail")
21
+ end
22
+
12
23
  raise
13
24
  end
14
25
 
26
+ def prepare(job_hash, &block)
27
+ level = job_hash["log_level"]
28
+ if level
29
+ @logger.log_at(level) do
30
+ Sidekiq::Context.with(job_hash_context(job_hash), &block)
31
+ end
32
+ else
33
+ Sidekiq::Context.with(job_hash_context(job_hash), &block)
34
+ end
35
+ end
36
+
37
+ def job_hash_context(job_hash)
38
+ # If we're using a wrapper class, like ActiveJob, use the "wrapped"
39
+ # attribute to expose the underlying thing.
40
+ h = {
41
+ class: job_hash["wrapped"] || job_hash["class"],
42
+ jid: job_hash["jid"]
43
+ }
44
+ h[:bid] = job_hash["bid"] if job_hash["bid"]
45
+ h[:tags] = job_hash["tags"] if job_hash["tags"]
46
+ h
47
+ end
48
+
49
+ def with_elapsed_time_context(start, &block)
50
+ Sidekiq::Context.with(elapsed_time_context(start), &block)
51
+ end
52
+
53
+ def elapsed_time_context(start)
54
+ {elapsed: elapsed(start).to_s}
55
+ end
56
+
15
57
  private
16
58
 
17
59
  def elapsed(start)
18
60
  (::Process.clock_gettime(::Process::CLOCK_MONOTONIC) - start).round(3)
19
61
  end
20
-
21
- def logger
22
- Sidekiq.logger
23
- end
24
62
  end
25
63
  end