sidekiq 5.2.7 → 6.0.0

Sign up to get free protection for your applications and to get access to all the features.

Potentially problematic release.


This version of sidekiq might be problematic. Click here for more details.

Files changed (61) hide show
  1. checksums.yaml +4 -4
  2. data/.gitignore +0 -2
  3. data/.standard.yml +20 -0
  4. data/6.0-Upgrade.md +70 -0
  5. data/Changes.md +34 -0
  6. data/Ent-2.0-Upgrade.md +37 -0
  7. data/Ent-Changes.md +12 -0
  8. data/Gemfile +12 -11
  9. data/Gemfile.lock +196 -0
  10. data/Pro-5.0-Upgrade.md +25 -0
  11. data/Pro-Changes.md +12 -3
  12. data/README.md +16 -30
  13. data/Rakefile +5 -4
  14. data/bin/sidekiqload +26 -22
  15. data/bin/sidekiqmon +9 -0
  16. data/lib/generators/sidekiq/templates/worker_test.rb.erb +1 -1
  17. data/lib/generators/sidekiq/worker_generator.rb +12 -14
  18. data/lib/sidekiq.rb +53 -42
  19. data/lib/sidekiq/api.rb +138 -151
  20. data/lib/sidekiq/cli.rb +97 -162
  21. data/lib/sidekiq/client.rb +45 -46
  22. data/lib/sidekiq/delay.rb +5 -6
  23. data/lib/sidekiq/exception_handler.rb +10 -12
  24. data/lib/sidekiq/extensions/action_mailer.rb +10 -20
  25. data/lib/sidekiq/extensions/active_record.rb +9 -7
  26. data/lib/sidekiq/extensions/class_methods.rb +9 -7
  27. data/lib/sidekiq/extensions/generic_proxy.rb +4 -4
  28. data/lib/sidekiq/fetch.rb +5 -6
  29. data/lib/sidekiq/job_logger.rb +37 -7
  30. data/lib/sidekiq/job_retry.rb +45 -58
  31. data/lib/sidekiq/launcher.rb +59 -51
  32. data/lib/sidekiq/logger.rb +69 -0
  33. data/lib/sidekiq/manager.rb +7 -9
  34. data/lib/sidekiq/middleware/chain.rb +3 -2
  35. data/lib/sidekiq/middleware/i18n.rb +5 -7
  36. data/lib/sidekiq/monitor.rb +148 -0
  37. data/lib/sidekiq/paginator.rb +11 -12
  38. data/lib/sidekiq/processor.rb +52 -49
  39. data/lib/sidekiq/rails.rb +23 -29
  40. data/lib/sidekiq/redis_connection.rb +31 -37
  41. data/lib/sidekiq/scheduled.rb +17 -19
  42. data/lib/sidekiq/testing.rb +22 -23
  43. data/lib/sidekiq/testing/inline.rb +2 -1
  44. data/lib/sidekiq/util.rb +17 -14
  45. data/lib/sidekiq/version.rb +2 -1
  46. data/lib/sidekiq/web.rb +41 -49
  47. data/lib/sidekiq/web/action.rb +14 -10
  48. data/lib/sidekiq/web/application.rb +60 -57
  49. data/lib/sidekiq/web/helpers.rb +66 -67
  50. data/lib/sidekiq/web/router.rb +17 -14
  51. data/lib/sidekiq/worker.rb +124 -97
  52. data/sidekiq.gemspec +16 -16
  53. data/web/assets/javascripts/dashboard.js +2 -21
  54. data/web/locales/ja.yml +2 -1
  55. metadata +19 -29
  56. data/.travis.yml +0 -11
  57. data/bin/sidekiqctl +0 -20
  58. data/lib/sidekiq/core_ext.rb +0 -1
  59. data/lib/sidekiq/ctl.rb +0 -221
  60. data/lib/sidekiq/logging.rb +0 -122
  61. data/lib/sidekiq/middleware/server/active_record.rb +0 -23
@@ -1,14 +1,15 @@
1
1
  # frozen_string_literal: true
2
- require 'sidekiq'
3
- require 'sidekiq/util'
4
- require 'sidekiq/api'
2
+
3
+ require "sidekiq"
4
+ require "sidekiq/util"
5
+ require "sidekiq/api"
5
6
 
6
7
  module Sidekiq
7
8
  module Scheduled
8
- SETS = %w(retry schedule)
9
+ SETS = %w[retry schedule]
9
10
 
10
11
  class Enq
11
- def enqueue_jobs(now=Time.now.to_f.to_s, sorted_sets=SETS)
12
+ def enqueue_jobs(now = Time.now.to_f.to_s, sorted_sets = SETS)
12
13
  # A job's "score" in Redis is the time at which it should be processed.
13
14
  # Just check Redis for the set of jobs with a timestamp before now.
14
15
  Sidekiq.redis do |conn|
@@ -17,14 +18,14 @@ module Sidekiq
17
18
  # We need to go through the list one at a time to reduce the risk of something
18
19
  # going wrong between the time jobs are popped from the scheduled queue and when
19
20
  # they are pushed onto a work queue and losing the jobs.
20
- while job = conn.zrangebyscore(sorted_set, '-inf', now, :limit => [0, 1]).first do
21
+ while (job = conn.zrangebyscore(sorted_set, "-inf", now, limit: [0, 1]).first)
21
22
 
22
23
  # Pop item off the queue and add it to the work queue. If the job can't be popped from
23
24
  # the queue, it's because another process already popped it so we can move on to the
24
25
  # next one.
25
26
  if conn.zrem(sorted_set, job)
26
27
  Sidekiq::Client.push(Sidekiq.load_json(job))
27
- Sidekiq::Logging.logger.debug { "enqueued #{sorted_set}: #{job}" }
28
+ Sidekiq.logger.debug { "enqueued #{sorted_set}: #{job}" }
28
29
  end
29
30
  end
30
31
  end
@@ -61,26 +62,24 @@ module Sidekiq
61
62
  end
62
63
 
63
64
  def start
64
- @thread ||= safe_thread("scheduler") do
65
+ @thread ||= safe_thread("scheduler") {
65
66
  initial_wait
66
67
 
67
- while !@done
68
+ until @done
68
69
  enqueue
69
70
  wait
70
71
  end
71
72
  Sidekiq.logger.info("Scheduler exiting...")
72
- end
73
+ }
73
74
  end
74
75
 
75
76
  def enqueue
76
- begin
77
- @enq.enqueue_jobs
78
- rescue => ex
79
- # Most likely a problem with redis networking.
80
- # Punt and try again at the next interval
81
- logger.error ex.message
82
- handle_exception(ex)
83
- end
77
+ @enq.enqueue_jobs
78
+ rescue => ex
79
+ # Most likely a problem with redis networking.
80
+ # Punt and try again at the next interval
81
+ logger.error ex.message
82
+ handle_exception(ex)
84
83
  end
85
84
 
86
85
  private
@@ -168,7 +167,6 @@ module Sidekiq
168
167
  @sleeper.pop(total)
169
168
  rescue Timeout::Error
170
169
  end
171
-
172
170
  end
173
171
  end
174
172
  end
@@ -1,16 +1,16 @@
1
1
  # frozen_string_literal: true
2
- require 'securerandom'
3
- require 'sidekiq'
4
2
 
5
- module Sidekiq
3
+ require "securerandom"
4
+ require "sidekiq"
6
5
 
6
+ module Sidekiq
7
7
  class Testing
8
8
  class << self
9
9
  attr_accessor :__test_mode
10
10
 
11
11
  def __set_test_mode(mode)
12
12
  if block_given?
13
- current_mode = self.__test_mode
13
+ current_mode = __test_mode
14
14
  begin
15
15
  self.__test_mode = mode
16
16
  yield
@@ -35,19 +35,19 @@ module Sidekiq
35
35
  end
36
36
 
37
37
  def enabled?
38
- self.__test_mode != :disable
38
+ __test_mode != :disable
39
39
  end
40
40
 
41
41
  def disabled?
42
- self.__test_mode == :disable
42
+ __test_mode == :disable
43
43
  end
44
44
 
45
45
  def fake?
46
- self.__test_mode == :fake
46
+ __test_mode == :fake
47
47
  end
48
48
 
49
49
  def inline?
50
- self.__test_mode == :inline
50
+ __test_mode == :inline
51
51
  end
52
52
 
53
53
  def server_middleware
@@ -57,7 +57,7 @@ module Sidekiq
57
57
  end
58
58
 
59
59
  def constantize(str)
60
- names = str.split('::')
60
+ names = str.split("::")
61
61
  names.shift if names.empty? || names.first.empty?
62
62
 
63
63
  names.inject(Object) do |constant, name|
@@ -77,14 +77,14 @@ module Sidekiq
77
77
  if Sidekiq::Testing.fake?
78
78
  payloads.each do |job|
79
79
  job = Sidekiq.load_json(Sidekiq.dump_json(job))
80
- job.merge!('enqueued_at' => Time.now.to_f) unless job['at']
81
- Queues.push(job['queue'], job['class'], job)
80
+ job["enqueued_at"] = Time.now.to_f unless job["at"]
81
+ Queues.push(job["queue"], job["class"], job)
82
82
  end
83
83
  true
84
84
  elsif Sidekiq::Testing.inline?
85
85
  payloads.each do |job|
86
- klass = Sidekiq::Testing.constantize(job['class'])
87
- job['id'] ||= SecureRandom.hex(12)
86
+ klass = Sidekiq::Testing.constantize(job["class"])
87
+ job["id"] ||= SecureRandom.hex(12)
88
88
  job_hash = Sidekiq.load_json(Sidekiq.dump_json(job))
89
89
  klass.process_job(job_hash)
90
90
  end
@@ -255,27 +255,26 @@ module Sidekiq
255
255
  # Then I should receive a welcome email to "foo@example.com"
256
256
  #
257
257
  module ClassMethods
258
-
259
258
  # Queue for this worker
260
259
  def queue
261
- self.sidekiq_options["queue"]
260
+ get_sidekiq_options["queue"]
262
261
  end
263
262
 
264
263
  # Jobs queued for this worker
265
264
  def jobs
266
- Queues.jobs_by_worker[self.to_s]
265
+ Queues.jobs_by_worker[to_s]
267
266
  end
268
267
 
269
268
  # Clear all jobs for this worker
270
269
  def clear
271
- Queues.clear_for(queue, self.to_s)
270
+ Queues.clear_for(queue, to_s)
272
271
  end
273
272
 
274
273
  # Drain and run all jobs for this worker
275
274
  def drain
276
275
  while jobs.any?
277
276
  next_job = jobs.first
278
- Queues.delete_for(next_job["jid"], next_job["queue"], self.to_s)
277
+ Queues.delete_for(next_job["jid"], next_job["queue"], to_s)
279
278
  process_job(next_job)
280
279
  end
281
280
  end
@@ -284,16 +283,16 @@ module Sidekiq
284
283
  def perform_one
285
284
  raise(EmptyQueueError, "perform_one called with empty job queue") if jobs.empty?
286
285
  next_job = jobs.first
287
- Queues.delete_for(next_job["jid"], queue, self.to_s)
286
+ Queues.delete_for(next_job["jid"], queue, to_s)
288
287
  process_job(next_job)
289
288
  end
290
289
 
291
290
  def process_job(job)
292
291
  worker = new
293
- worker.jid = job['jid']
294
- worker.bid = job['bid'] if worker.respond_to?(:bid=)
295
- Sidekiq::Testing.server_middleware.invoke(worker, job, job['queue']) do
296
- execute_job(worker, job['args'])
292
+ worker.jid = job["jid"]
293
+ worker.bid = job["bid"] if worker.respond_to?(:bid=)
294
+ Sidekiq::Testing.server_middleware.invoke(worker, job, job["queue"]) do
295
+ execute_job(worker, job["args"])
297
296
  end
298
297
  end
299
298
 
@@ -1,5 +1,6 @@
1
1
  # frozen_string_literal: true
2
- require 'sidekiq/testing'
2
+
3
+ require "sidekiq/testing"
3
4
 
4
5
  ##
5
6
  # The Sidekiq inline infrastructure overrides perform_async so that it
@@ -1,7 +1,8 @@
1
1
  # frozen_string_literal: true
2
- require 'socket'
3
- require 'securerandom'
4
- require 'sidekiq/exception_handler'
2
+
3
+ require "socket"
4
+ require "securerandom"
5
+ require "sidekiq/exception_handler"
5
6
 
6
7
  module Sidekiq
7
8
  ##
@@ -15,13 +16,13 @@ module Sidekiq
15
16
  def watchdog(last_words)
16
17
  yield
17
18
  rescue Exception => ex
18
- handle_exception(ex, { context: last_words })
19
+ handle_exception(ex, {context: last_words})
19
20
  raise ex
20
21
  end
21
22
 
22
23
  def safe_thread(name, &block)
23
24
  Thread.new do
24
- Thread.current['sidekiq_label'] = name
25
+ Thread.current.name = name
25
26
  watchdog(name, &block)
26
27
  end
27
28
  end
@@ -34,8 +35,12 @@ module Sidekiq
34
35
  Sidekiq.redis(&block)
35
36
  end
36
37
 
38
+ def tid
39
+ Thread.current["sidekiq_tid"] ||= (Thread.current.object_id ^ ::Process.pid).to_s(36)
40
+ end
41
+
37
42
  def hostname
38
- ENV['DYNO'] || Socket.gethostname
43
+ ENV["DYNO"] || Socket.gethostname
39
44
  end
40
45
 
41
46
  def process_nonce
@@ -43,22 +48,20 @@ module Sidekiq
43
48
  end
44
49
 
45
50
  def identity
46
- @@identity ||= "#{hostname}:#{$$}:#{process_nonce}"
51
+ @@identity ||= "#{hostname}:#{::Process.pid}:#{process_nonce}"
47
52
  end
48
53
 
49
- def fire_event(event, options={})
54
+ def fire_event(event, options = {})
50
55
  reverse = options[:reverse]
51
56
  reraise = options[:reraise]
52
57
 
53
58
  arr = Sidekiq.options[:lifecycle_events][event]
54
59
  arr.reverse! if reverse
55
60
  arr.each do |block|
56
- begin
57
- block.call
58
- rescue => ex
59
- handle_exception(ex, { context: "Exception during Sidekiq lifecycle event.", event: event })
60
- raise ex if reraise
61
- end
61
+ block.call
62
+ rescue => ex
63
+ handle_exception(ex, {context: "Exception during Sidekiq lifecycle event.", event: event})
64
+ raise ex if reraise
62
65
  end
63
66
  arr.clear
64
67
  end
@@ -1,4 +1,5 @@
1
1
  # frozen_string_literal: true
2
+
2
3
  module Sidekiq
3
- VERSION = "5.2.7"
4
+ VERSION = "6.0.0"
4
5
  end
@@ -1,20 +1,21 @@
1
1
  # frozen_string_literal: true
2
- require 'erb'
3
2
 
4
- require 'sidekiq'
5
- require 'sidekiq/api'
6
- require 'sidekiq/paginator'
7
- require 'sidekiq/web/helpers'
3
+ require "erb"
8
4
 
9
- require 'sidekiq/web/router'
10
- require 'sidekiq/web/action'
11
- require 'sidekiq/web/application'
5
+ require "sidekiq"
6
+ require "sidekiq/api"
7
+ require "sidekiq/paginator"
8
+ require "sidekiq/web/helpers"
12
9
 
13
- require 'rack/protection'
10
+ require "sidekiq/web/router"
11
+ require "sidekiq/web/action"
12
+ require "sidekiq/web/application"
14
13
 
15
- require 'rack/builder'
16
- require 'rack/file'
17
- require 'rack/session/cookie'
14
+ require "rack/protection"
15
+
16
+ require "rack/builder"
17
+ require "rack/file"
18
+ require "rack/session/cookie"
18
19
 
19
20
  module Sidekiq
20
21
  class Web
@@ -25,12 +26,12 @@ module Sidekiq
25
26
  ASSETS = "#{ROOT}/assets"
26
27
 
27
28
  DEFAULT_TABS = {
28
- "Dashboard" => '',
29
- "Busy" => 'busy',
30
- "Queues" => 'queues',
31
- "Retries" => 'retries',
32
- "Scheduled" => 'scheduled',
33
- "Dead" => 'morgue',
29
+ "Dashboard" => "",
30
+ "Busy" => "busy",
31
+ "Queues" => "queues",
32
+ "Retries" => "retries",
33
+ "Scheduled" => "scheduled",
34
+ "Dead" => "morgue",
34
35
  }
35
36
 
36
37
  class << self
@@ -64,11 +65,11 @@ module Sidekiq
64
65
  end
65
66
 
66
67
  def enable(*opts)
67
- opts.each {|key| set(key, true) }
68
+ opts.each { |key| set(key, true) }
68
69
  end
69
70
 
70
71
  def disable(*opts)
71
- opts.each {|key| set(key, false) }
72
+ opts.each { |key| set(key, false) }
72
73
  end
73
74
 
74
75
  # Helper for the Sinatra syntax: Sidekiq::Web.set(:session_secret, Rails.application.secrets...)
@@ -81,10 +82,10 @@ module Sidekiq
81
82
  end
82
83
 
83
84
  def self.inherited(child)
84
- child.app_url = self.app_url
85
- child.session_secret = self.session_secret
86
- child.redis_pool = self.redis_pool
87
- child.sessions = self.sessions
85
+ child.app_url = app_url
86
+ child.session_secret = session_secret
87
+ child.redis_pool = redis_pool
88
+ child.sessions = sessions
88
89
  end
89
90
 
90
91
  def settings
@@ -113,11 +114,11 @@ module Sidekiq
113
114
  end
114
115
 
115
116
  def enable(*opts)
116
- opts.each {|key| set(key, true) }
117
+ opts.each { |key| set(key, true) }
117
118
  end
118
119
 
119
120
  def disable(*opts)
120
- opts.each {|key| set(key, false) }
121
+ opts.each { |key| set(key, false) }
121
122
  end
122
123
 
123
124
  def set(attribute, value)
@@ -145,28 +146,28 @@ module Sidekiq
145
146
  private
146
147
 
147
148
  def using?(middleware)
148
- middlewares.any? do |(m,_)|
149
- m.kind_of?(Array) && (m[0] == middleware || m[0].kind_of?(middleware))
149
+ middlewares.any? do |(m, _)|
150
+ m.is_a?(Array) && (m[0] == middleware || m[0].is_a?(middleware))
150
151
  end
151
152
  end
152
153
 
153
154
  def build_sessions
154
155
  middlewares = self.middlewares
155
156
 
156
- unless using?(::Rack::Protection) || ENV['RACK_ENV'] == 'test'
157
- middlewares.unshift [[::Rack::Protection, { use: :authenticity_token }], nil]
157
+ unless using?(::Rack::Protection) || ENV["RACK_ENV"] == "test"
158
+ middlewares.unshift [[::Rack::Protection, {use: :authenticity_token}], nil]
158
159
  end
159
160
 
160
161
  s = sessions
161
162
  return unless s
162
163
 
163
164
  unless using? ::Rack::Session::Cookie
164
- unless secret = Web.session_secret
165
- require 'securerandom'
165
+ unless (secret = Web.session_secret)
166
+ require "securerandom"
166
167
  secret = SecureRandom.hex(64)
167
168
  end
168
169
 
169
- options = { secret: secret }
170
+ options = {secret: secret}
170
171
  options = options.merge(s.to_hash) if s.respond_to? :to_hash
171
172
 
172
173
  middlewares.unshift [[::Rack::Session::Cookie, options], nil]
@@ -180,13 +181,13 @@ module Sidekiq
180
181
  klass = self.class
181
182
 
182
183
  ::Rack::Builder.new do
183
- %w(stylesheets javascripts images).each do |asset_dir|
184
+ %w[stylesheets javascripts images].each do |asset_dir|
184
185
  map "/#{asset_dir}" do
185
- run ::Rack::File.new("#{ASSETS}/#{asset_dir}", { 'Cache-Control' => 'public, max-age=86400' })
186
+ run ::Rack::File.new("#{ASSETS}/#{asset_dir}", {"Cache-Control" => "public, max-age=86400"})
186
187
  end
187
188
  end
188
189
 
189
- middlewares.each {|middleware, block| use(*middleware, &block) }
190
+ middlewares.each { |middleware, block| use(*middleware, &block) }
190
191
 
191
192
  run WebApplication.new(klass)
192
193
  end
@@ -196,18 +197,9 @@ module Sidekiq
196
197
  Sidekiq::WebApplication.helpers WebHelpers
197
198
  Sidekiq::WebApplication.helpers Sidekiq::Paginator
198
199
 
199
- Sidekiq::WebAction.class_eval "def _render\n#{ERB.new(File.read(Web::LAYOUT)).src}\nend"
200
- end
201
-
202
- if defined?(::ActionDispatch::Request::Session) &&
203
- !::ActionDispatch::Request::Session.method_defined?(:each)
204
- # mperham/sidekiq#2460
205
- # Rack apps can't reuse the Rails session store without
206
- # this monkeypatch, fixed in Rails 5.
207
- class ActionDispatch::Request::Session
208
- def each(&block)
209
- hash = self.to_hash
210
- hash.each(&block)
200
+ Sidekiq::WebAction.class_eval <<-RUBY, __FILE__, __LINE__ + 1
201
+ def _render
202
+ #{ERB.new(File.read(Web::LAYOUT)).src}
211
203
  end
212
- end
204
+ RUBY
213
205
  end