sidekiq 5.2.3 → 6.0.0

Sign up to get free protection for your applications and to get access to all the features.

Potentially problematic release.


This version of sidekiq might be problematic. Click here for more details.

Files changed (64) hide show
  1. checksums.yaml +5 -5
  2. data/.circleci/config.yml +61 -0
  3. data/.gitignore +1 -1
  4. data/.standard.yml +20 -0
  5. data/6.0-Upgrade.md +70 -0
  6. data/COMM-LICENSE +11 -9
  7. data/Changes.md +61 -0
  8. data/Ent-2.0-Upgrade.md +37 -0
  9. data/Ent-Changes.md +27 -1
  10. data/Gemfile +19 -9
  11. data/Gemfile.lock +196 -0
  12. data/Pro-5.0-Upgrade.md +25 -0
  13. data/Pro-Changes.md +19 -2
  14. data/README.md +17 -31
  15. data/Rakefile +6 -4
  16. data/bin/sidekiqload +27 -23
  17. data/bin/sidekiqmon +9 -0
  18. data/lib/generators/sidekiq/templates/worker_test.rb.erb +1 -1
  19. data/lib/generators/sidekiq/worker_generator.rb +12 -14
  20. data/lib/sidekiq.rb +56 -43
  21. data/lib/sidekiq/api.rb +138 -151
  22. data/lib/sidekiq/cli.rb +141 -206
  23. data/lib/sidekiq/client.rb +45 -46
  24. data/lib/sidekiq/delay.rb +5 -6
  25. data/lib/sidekiq/exception_handler.rb +10 -12
  26. data/lib/sidekiq/extensions/action_mailer.rb +10 -20
  27. data/lib/sidekiq/extensions/active_record.rb +9 -7
  28. data/lib/sidekiq/extensions/class_methods.rb +9 -7
  29. data/lib/sidekiq/extensions/generic_proxy.rb +4 -4
  30. data/lib/sidekiq/fetch.rb +5 -6
  31. data/lib/sidekiq/job_logger.rb +37 -7
  32. data/lib/sidekiq/job_retry.rb +55 -57
  33. data/lib/sidekiq/launcher.rb +59 -51
  34. data/lib/sidekiq/logger.rb +69 -0
  35. data/lib/sidekiq/manager.rb +7 -9
  36. data/lib/sidekiq/middleware/chain.rb +3 -2
  37. data/lib/sidekiq/middleware/i18n.rb +5 -7
  38. data/lib/sidekiq/monitor.rb +148 -0
  39. data/lib/sidekiq/paginator.rb +11 -12
  40. data/lib/sidekiq/processor.rb +68 -58
  41. data/lib/sidekiq/rails.rb +24 -29
  42. data/lib/sidekiq/redis_connection.rb +31 -37
  43. data/lib/sidekiq/scheduled.rb +17 -19
  44. data/lib/sidekiq/testing.rb +22 -23
  45. data/lib/sidekiq/testing/inline.rb +2 -1
  46. data/lib/sidekiq/util.rb +17 -14
  47. data/lib/sidekiq/version.rb +2 -1
  48. data/lib/sidekiq/web.rb +41 -49
  49. data/lib/sidekiq/web/action.rb +14 -10
  50. data/lib/sidekiq/web/application.rb +61 -58
  51. data/lib/sidekiq/web/helpers.rb +72 -66
  52. data/lib/sidekiq/web/router.rb +17 -14
  53. data/lib/sidekiq/worker.rb +134 -102
  54. data/sidekiq.gemspec +16 -18
  55. data/web/assets/javascripts/dashboard.js +2 -21
  56. data/web/assets/stylesheets/bootstrap.css +1 -1
  57. data/web/locales/ja.yml +2 -1
  58. data/web/views/queues.erb +1 -1
  59. metadata +31 -26
  60. data/.travis.yml +0 -14
  61. data/bin/sidekiqctl +0 -237
  62. data/lib/sidekiq/core_ext.rb +0 -1
  63. data/lib/sidekiq/logging.rb +0 -122
  64. data/lib/sidekiq/middleware/server/active_record.rb +0 -23
@@ -1,10 +1,10 @@
1
1
  # frozen_string_literal: true
2
- require 'securerandom'
3
- require 'sidekiq/middleware/chain'
2
+
3
+ require "securerandom"
4
+ require "sidekiq/middleware/chain"
4
5
 
5
6
  module Sidekiq
6
7
  class Client
7
-
8
8
  ##
9
9
  # Define client-side middleware:
10
10
  #
@@ -38,7 +38,7 @@ module Sidekiq
38
38
  # Generally this is only needed for very large Sidekiq installs processing
39
39
  # thousands of jobs per second. I don't recommend sharding unless you
40
40
  # cannot scale any other way (e.g. splitting your app into smaller apps).
41
- def initialize(redis_pool=nil)
41
+ def initialize(redis_pool = nil)
42
42
  @redis_pool = redis_pool || Thread.current[:sidekiq_via_pool] || Sidekiq.redis_pool
43
43
  end
44
44
 
@@ -68,11 +68,11 @@ module Sidekiq
68
68
  #
69
69
  def push(item)
70
70
  normed = normalize_item(item)
71
- payload = process_single(item['class'], normed)
71
+ payload = process_single(item["class"], normed)
72
72
 
73
73
  if payload
74
74
  raw_push([payload])
75
- payload['jid']
75
+ payload["jid"]
76
76
  end
77
77
  end
78
78
 
@@ -90,19 +90,19 @@ module Sidekiq
90
90
  # Returns an array of the of pushed jobs' jids. The number of jobs pushed can be less
91
91
  # than the number given if the middleware stopped processing for one or more jobs.
92
92
  def push_bulk(items)
93
- arg = items['args'].first
93
+ arg = items["args"].first
94
94
  return [] unless arg # no jobs to push
95
- raise ArgumentError, "Bulk arguments must be an Array of Arrays: [[1], [2]]" if !arg.is_a?(Array)
95
+ raise ArgumentError, "Bulk arguments must be an Array of Arrays: [[1], [2]]" unless arg.is_a?(Array)
96
96
 
97
97
  normed = normalize_item(items)
98
- payloads = items['args'].map do |args|
99
- copy = normed.merge('args' => args, 'jid' => SecureRandom.hex(12), 'enqueued_at' => Time.now.to_f)
100
- result = process_single(items['class'], copy)
101
- result ? result : nil
102
- end.compact
103
-
104
- raw_push(payloads) if !payloads.empty?
105
- payloads.collect { |payload| payload['jid'] }
98
+ payloads = items["args"].map { |args|
99
+ copy = normed.merge("args" => args, "jid" => SecureRandom.hex(12), "enqueued_at" => Time.now.to_f)
100
+ result = process_single(items["class"], copy)
101
+ result || nil
102
+ }.compact
103
+
104
+ raw_push(payloads) unless payloads.empty?
105
+ payloads.collect { |payload| payload["jid"] }
106
106
  end
107
107
 
108
108
  # Allows sharding of jobs across any number of Redis instances. All jobs
@@ -127,7 +127,6 @@ module Sidekiq
127
127
  end
128
128
 
129
129
  class << self
130
-
131
130
  def push(item)
132
131
  new.push(item)
133
132
  end
@@ -145,14 +144,14 @@ module Sidekiq
145
144
  # Messages are enqueued to the 'default' queue.
146
145
  #
147
146
  def enqueue(klass, *args)
148
- klass.client_push('class' => klass, 'args' => args)
147
+ klass.client_push("class" => klass, "args" => args)
149
148
  end
150
149
 
151
150
  # Example usage:
152
151
  # Sidekiq::Client.enqueue_to(:queue_name, MyWorker, 'foo', 1, :bat => 'bar')
153
152
  #
154
153
  def enqueue_to(queue, klass, *args)
155
- klass.client_push('queue' => queue, 'class' => klass, 'args' => args)
154
+ klass.client_push("queue" => queue, "class" => klass, "args" => args)
156
155
  end
157
156
 
158
157
  # Example usage:
@@ -163,8 +162,8 @@ module Sidekiq
163
162
  now = Time.now.to_f
164
163
  ts = (int < 1_000_000_000 ? now + int : int)
165
164
 
166
- item = { 'class' => klass, 'args' => args, 'at' => ts, 'queue' => queue }
167
- item.delete('at') if ts <= now
165
+ item = {"class" => klass, "args" => args, "at" => ts, "queue" => queue}
166
+ item.delete("at") if ts <= now
168
167
 
169
168
  klass.client_push(item)
170
169
  end
@@ -189,25 +188,25 @@ module Sidekiq
189
188
  end
190
189
 
191
190
  def atomic_push(conn, payloads)
192
- if payloads.first['at']
193
- conn.zadd('schedule', payloads.map do |hash|
194
- at = hash.delete('at').to_s
191
+ if payloads.first["at"]
192
+ conn.zadd("schedule", payloads.map { |hash|
193
+ at = hash.delete("at").to_s
195
194
  [at, Sidekiq.dump_json(hash)]
196
- end)
195
+ })
197
196
  else
198
- q = payloads.first['queue']
197
+ queue = payloads.first["queue"]
199
198
  now = Time.now.to_f
200
- to_push = payloads.map do |entry|
201
- entry['enqueued_at'] = now
199
+ to_push = payloads.map { |entry|
200
+ entry["enqueued_at"] = now
202
201
  Sidekiq.dump_json(entry)
203
- end
204
- conn.sadd('queues', q)
205
- conn.lpush("queue:#{q}", to_push)
202
+ }
203
+ conn.sadd("queues", queue)
204
+ conn.lpush("queue:#{queue}", to_push)
206
205
  end
207
206
  end
208
207
 
209
208
  def process_single(worker_class, item)
210
- queue = item['queue']
209
+ queue = item["queue"]
211
210
 
212
211
  middleware.invoke(worker_class, item, queue, @redis_pool) do
213
212
  item
@@ -215,25 +214,25 @@ module Sidekiq
215
214
  end
216
215
 
217
216
  def normalize_item(item)
218
- raise(ArgumentError, "Job must be a Hash with 'class' and 'args' keys: { 'class' => SomeWorker, 'args' => ['bob', 1, :foo => 'bar'] }") unless item.is_a?(Hash) && item.has_key?('class') && item.has_key?('args')
219
- raise(ArgumentError, "Job args must be an Array") unless item['args'].is_a?(Array)
220
- raise(ArgumentError, "Job class must be either a Class or String representation of the class name") unless item['class'].is_a?(Class) || item['class'].is_a?(String)
221
- raise(ArgumentError, "Job 'at' must be a Numeric timestamp") if item.has_key?('at') && !item['at'].is_a?(Numeric)
222
- #raise(ArgumentError, "Arguments must be native JSON types, see https://github.com/mperham/sidekiq/wiki/Best-Practices") unless JSON.load(JSON.dump(item['args'])) == item['args']
223
-
224
- normalized_hash(item['class'])
225
- .each{ |key, value| item[key] = value if item[key].nil? }
226
-
227
- item['class'] = item['class'].to_s
228
- item['queue'] = item['queue'].to_s
229
- item['jid'] ||= SecureRandom.hex(12)
230
- item['created_at'] ||= Time.now.to_f
217
+ raise(ArgumentError, "Job must be a Hash with 'class' and 'args' keys: { 'class' => SomeWorker, 'args' => ['bob', 1, :foo => 'bar'] }") unless item.is_a?(Hash) && item.key?("class") && item.key?("args")
218
+ raise(ArgumentError, "Job args must be an Array") unless item["args"].is_a?(Array)
219
+ raise(ArgumentError, "Job class must be either a Class or String representation of the class name") unless item["class"].is_a?(Class) || item["class"].is_a?(String)
220
+ raise(ArgumentError, "Job 'at' must be a Numeric timestamp") if item.key?("at") && !item["at"].is_a?(Numeric)
221
+ # raise(ArgumentError, "Arguments must be native JSON types, see https://github.com/mperham/sidekiq/wiki/Best-Practices") unless JSON.load(JSON.dump(item['args'])) == item['args']
222
+
223
+ normalized_hash(item["class"])
224
+ .each { |key, value| item[key] = value if item[key].nil? }
225
+
226
+ item["class"] = item["class"].to_s
227
+ item["queue"] = item["queue"].to_s
228
+ item["jid"] ||= SecureRandom.hex(12)
229
+ item["created_at"] ||= Time.now.to_f
231
230
  item
232
231
  end
233
232
 
234
233
  def normalized_hash(item_class)
235
234
  if item_class.is_a?(Class)
236
- raise(ArgumentError, "Message must include a Sidekiq::Worker class, not class name: #{item_class.ancestors.inspect}") if !item_class.respond_to?('get_sidekiq_options')
235
+ raise(ArgumentError, "Message must include a Sidekiq::Worker class, not class name: #{item_class.ancestors.inspect}") unless item_class.respond_to?("get_sidekiq_options")
237
236
  item_class.get_sidekiq_options
238
237
  else
239
238
  Sidekiq.default_worker_options
@@ -1,11 +1,11 @@
1
1
  # frozen_string_literal: true
2
+
2
3
  module Sidekiq
3
4
  module Extensions
4
-
5
5
  def self.enable_delay!
6
6
  if defined?(::ActiveSupport)
7
- require 'sidekiq/extensions/active_record'
8
- require 'sidekiq/extensions/action_mailer'
7
+ require "sidekiq/extensions/active_record"
8
+ require "sidekiq/extensions/action_mailer"
9
9
 
10
10
  # Need to patch Psych so it can autoload classes whose names are serialized
11
11
  # in the delayed YAML.
@@ -19,7 +19,7 @@ module Sidekiq
19
19
  end
20
20
  end
21
21
 
22
- require 'sidekiq/extensions/class_methods'
22
+ require "sidekiq/extensions/class_methods"
23
23
  Module.__send__(:include, Sidekiq::Extensions::Klass)
24
24
  end
25
25
 
@@ -27,7 +27,7 @@ module Sidekiq
27
27
  def resolve_class(klass_name)
28
28
  return nil if !klass_name || klass_name.empty?
29
29
  # constantize
30
- names = klass_name.split('::')
30
+ names = klass_name.split("::")
31
31
  names.shift if names.empty? || names.first.empty?
32
32
 
33
33
  names.inject(Object) do |constant, name|
@@ -39,4 +39,3 @@ module Sidekiq
39
39
  end
40
40
  end
41
41
  end
42
-
@@ -1,12 +1,12 @@
1
1
  # frozen_string_literal: true
2
- require 'sidekiq'
2
+
3
+ require "sidekiq"
3
4
 
4
5
  module Sidekiq
5
6
  module ExceptionHandler
6
-
7
7
  class Logger
8
- def call(ex, ctxHash)
9
- Sidekiq.logger.warn(Sidekiq.dump_json(ctxHash)) if !ctxHash.empty?
8
+ def call(ex, ctx)
9
+ Sidekiq.logger.warn(Sidekiq.dump_json(ctx)) unless ctx.empty?
10
10
  Sidekiq.logger.warn("#{ex.class.name}: #{ex.message}")
11
11
  Sidekiq.logger.warn(ex.backtrace.join("\n")) unless ex.backtrace.nil?
12
12
  end
@@ -14,15 +14,13 @@ module Sidekiq
14
14
  Sidekiq.error_handlers << Sidekiq::ExceptionHandler::Logger.new
15
15
  end
16
16
 
17
- def handle_exception(ex, ctxHash={})
17
+ def handle_exception(ex, ctx = {})
18
18
  Sidekiq.error_handlers.each do |handler|
19
- begin
20
- handler.call(ex, ctxHash)
21
- rescue => ex
22
- Sidekiq.logger.error "!!! ERROR HANDLER THREW AN ERROR !!!"
23
- Sidekiq.logger.error ex
24
- Sidekiq.logger.error ex.backtrace.join("\n") unless ex.backtrace.nil?
25
- end
19
+ handler.call(ex, ctx)
20
+ rescue => ex
21
+ Sidekiq.logger.error "!!! ERROR HANDLER THREW AN ERROR !!!"
22
+ Sidekiq.logger.error ex
23
+ Sidekiq.logger.error ex.backtrace.join("\n") unless ex.backtrace.nil?
26
24
  end
27
25
  end
28
26
  end
@@ -1,5 +1,6 @@
1
1
  # frozen_string_literal: true
2
- require 'sidekiq/extensions/generic_proxy'
2
+
3
+ require "sidekiq/extensions/generic_proxy"
3
4
 
4
5
  module Sidekiq
5
6
  module Extensions
@@ -19,39 +20,28 @@ module Sidekiq
19
20
  # The email method can return nil, which causes ActionMailer to return
20
21
  # an undeliverable empty message.
21
22
  if msg
22
- deliver(msg)
23
- else
24
- raise "#{target.name}##{method_name} returned an undeliverable mail object"
25
- end
26
- end
27
-
28
- private
29
-
30
- def deliver(msg)
31
- if msg.respond_to?(:deliver_now)
32
- # Rails 4.2/5.0
33
23
  msg.deliver_now
34
24
  else
35
- # Rails 3.2/4.0/4.1
36
- msg.deliver
25
+ raise "#{target.name}##{method_name} returned an undeliverable mail object"
37
26
  end
38
27
  end
39
28
  end
40
29
 
41
30
  module ActionMailer
42
- def sidekiq_delay(options={})
31
+ def sidekiq_delay(options = {})
43
32
  Proxy.new(DelayedMailer, self, options)
44
33
  end
45
- def sidekiq_delay_for(interval, options={})
46
- Proxy.new(DelayedMailer, self, options.merge('at' => Time.now.to_f + interval.to_f))
34
+
35
+ def sidekiq_delay_for(interval, options = {})
36
+ Proxy.new(DelayedMailer, self, options.merge("at" => Time.now.to_f + interval.to_f))
47
37
  end
48
- def sidekiq_delay_until(timestamp, options={})
49
- Proxy.new(DelayedMailer, self, options.merge('at' => timestamp.to_f))
38
+
39
+ def sidekiq_delay_until(timestamp, options = {})
40
+ Proxy.new(DelayedMailer, self, options.merge("at" => timestamp.to_f))
50
41
  end
51
42
  alias_method :delay, :sidekiq_delay
52
43
  alias_method :delay_for, :sidekiq_delay_for
53
44
  alias_method :delay_until, :sidekiq_delay_until
54
45
  end
55
-
56
46
  end
57
47
  end
@@ -1,5 +1,6 @@
1
1
  # frozen_string_literal: true
2
- require 'sidekiq/extensions/generic_proxy'
2
+
3
+ require "sidekiq/extensions/generic_proxy"
3
4
 
4
5
  module Sidekiq
5
6
  module Extensions
@@ -22,19 +23,20 @@ module Sidekiq
22
23
  end
23
24
 
24
25
  module ActiveRecord
25
- def sidekiq_delay(options={})
26
+ def sidekiq_delay(options = {})
26
27
  Proxy.new(DelayedModel, self, options)
27
28
  end
28
- def sidekiq_delay_for(interval, options={})
29
- Proxy.new(DelayedModel, self, options.merge('at' => Time.now.to_f + interval.to_f))
29
+
30
+ def sidekiq_delay_for(interval, options = {})
31
+ Proxy.new(DelayedModel, self, options.merge("at" => Time.now.to_f + interval.to_f))
30
32
  end
31
- def sidekiq_delay_until(timestamp, options={})
32
- Proxy.new(DelayedModel, self, options.merge('at' => timestamp.to_f))
33
+
34
+ def sidekiq_delay_until(timestamp, options = {})
35
+ Proxy.new(DelayedModel, self, options.merge("at" => timestamp.to_f))
33
36
  end
34
37
  alias_method :delay, :sidekiq_delay
35
38
  alias_method :delay_for, :sidekiq_delay_for
36
39
  alias_method :delay_until, :sidekiq_delay_until
37
40
  end
38
-
39
41
  end
40
42
  end
@@ -1,5 +1,6 @@
1
1
  # frozen_string_literal: true
2
- require 'sidekiq/extensions/generic_proxy'
2
+
3
+ require "sidekiq/extensions/generic_proxy"
3
4
 
4
5
  module Sidekiq
5
6
  module Extensions
@@ -20,20 +21,21 @@ module Sidekiq
20
21
  end
21
22
 
22
23
  module Klass
23
- def sidekiq_delay(options={})
24
+ def sidekiq_delay(options = {})
24
25
  Proxy.new(DelayedClass, self, options)
25
26
  end
26
- def sidekiq_delay_for(interval, options={})
27
- Proxy.new(DelayedClass, self, options.merge('at' => Time.now.to_f + interval.to_f))
27
+
28
+ def sidekiq_delay_for(interval, options = {})
29
+ Proxy.new(DelayedClass, self, options.merge("at" => Time.now.to_f + interval.to_f))
28
30
  end
29
- def sidekiq_delay_until(timestamp, options={})
30
- Proxy.new(DelayedClass, self, options.merge('at' => timestamp.to_f))
31
+
32
+ def sidekiq_delay_until(timestamp, options = {})
33
+ Proxy.new(DelayedClass, self, options.merge("at" => timestamp.to_f))
31
34
  end
32
35
  alias_method :delay, :sidekiq_delay
33
36
  alias_method :delay_for, :sidekiq_delay_for
34
37
  alias_method :delay_until, :sidekiq_delay_until
35
38
  end
36
-
37
39
  end
38
40
  end
39
41
 
@@ -1,12 +1,13 @@
1
1
  # frozen_string_literal: true
2
- require 'yaml'
2
+
3
+ require "yaml"
3
4
 
4
5
  module Sidekiq
5
6
  module Extensions
6
7
  SIZE_LIMIT = 8_192
7
8
 
8
9
  class Proxy < BasicObject
9
- def initialize(performable, target, options={})
10
+ def initialize(performable, target, options = {})
10
11
  @performable = performable
11
12
  @target = target
12
13
  @opts = options
@@ -23,9 +24,8 @@ module Sidekiq
23
24
  if marshalled.size > SIZE_LIMIT
24
25
  ::Sidekiq.logger.warn { "#{@target}.#{name} job argument is #{marshalled.bytesize} bytes, you should refactor it to reduce the size" }
25
26
  end
26
- @performable.client_push({ 'class' => @performable, 'args' => [marshalled] }.merge(@opts))
27
+ @performable.client_push({"class" => @performable, "args" => [marshalled]}.merge(@opts))
27
28
  end
28
29
  end
29
-
30
30
  end
31
31
  end
@@ -1,5 +1,6 @@
1
1
  # frozen_string_literal: true
2
- require 'sidekiq'
2
+
3
+ require "sidekiq"
3
4
 
4
5
  module Sidekiq
5
6
  class BasicFetch
@@ -7,13 +8,13 @@ module Sidekiq
7
8
  # can check if the process is shutting down.
8
9
  TIMEOUT = 2
9
10
 
10
- UnitOfWork = Struct.new(:queue, :job) do
11
+ UnitOfWork = Struct.new(:queue, :job) {
11
12
  def acknowledge
12
13
  # nothing to do
13
14
  end
14
15
 
15
16
  def queue_name
16
- queue.sub(/.*queue:/, '')
17
+ queue.sub(/.*queue:/, "")
17
18
  end
18
19
 
19
20
  def requeue
@@ -21,7 +22,7 @@ module Sidekiq
21
22
  conn.rpush("queue:#{queue_name}", job)
22
23
  end
23
24
  end
24
- end
25
+ }
25
26
 
26
27
  def initialize(options)
27
28
  @strictly_ordered_queues = !!options[:strict]
@@ -52,7 +53,6 @@ module Sidekiq
52
53
  end
53
54
  end
54
55
 
55
-
56
56
  # By leaving this as a class method, it can be pluggable and used by the Manager actor. Making it
57
57
  # an instance method will make it async to the Fetcher actor
58
58
  def self.bulk_requeue(inprogress, options)
@@ -76,6 +76,5 @@ module Sidekiq
76
76
  rescue => ex
77
77
  Sidekiq.logger.warn("Failed to requeue #{inprogress.size} jobs: #{ex.message}")
78
78
  end
79
-
80
79
  end
81
80
  end