sidekiq 4.2.10 → 5.2.7

Sign up to get free protection for your applications and to get access to all the features.

Potentially problematic release.


This version of sidekiq might be problematic. Click here for more details.

Files changed (75) hide show
  1. checksums.yaml +5 -5
  2. data/.circleci/config.yml +61 -0
  3. data/.github/issue_template.md +3 -1
  4. data/.gitignore +3 -0
  5. data/.travis.yml +6 -13
  6. data/5.0-Upgrade.md +56 -0
  7. data/COMM-LICENSE +12 -10
  8. data/Changes.md +158 -1
  9. data/Ent-Changes.md +67 -2
  10. data/Gemfile +14 -20
  11. data/LICENSE +1 -1
  12. data/Pro-4.0-Upgrade.md +35 -0
  13. data/Pro-Changes.md +133 -2
  14. data/README.md +8 -6
  15. data/Rakefile +2 -5
  16. data/bin/sidekiqctl +13 -92
  17. data/bin/sidekiqload +5 -10
  18. data/lib/generators/sidekiq/templates/worker_spec.rb.erb +1 -1
  19. data/lib/sidekiq.rb +27 -27
  20. data/lib/sidekiq/api.rb +145 -57
  21. data/lib/sidekiq/cli.rb +120 -81
  22. data/lib/sidekiq/client.rb +25 -18
  23. data/lib/sidekiq/core_ext.rb +1 -119
  24. data/lib/sidekiq/ctl.rb +221 -0
  25. data/lib/sidekiq/delay.rb +42 -0
  26. data/lib/sidekiq/exception_handler.rb +2 -4
  27. data/lib/sidekiq/extensions/generic_proxy.rb +7 -1
  28. data/lib/sidekiq/fetch.rb +1 -1
  29. data/lib/sidekiq/job_logger.rb +25 -0
  30. data/lib/sidekiq/job_retry.rb +262 -0
  31. data/lib/sidekiq/launcher.rb +19 -19
  32. data/lib/sidekiq/logging.rb +18 -2
  33. data/lib/sidekiq/manager.rb +5 -6
  34. data/lib/sidekiq/middleware/server/active_record.rb +10 -0
  35. data/lib/sidekiq/processor.rb +126 -48
  36. data/lib/sidekiq/rails.rb +8 -73
  37. data/lib/sidekiq/redis_connection.rb +43 -5
  38. data/lib/sidekiq/scheduled.rb +35 -8
  39. data/lib/sidekiq/testing.rb +16 -7
  40. data/lib/sidekiq/util.rb +5 -2
  41. data/lib/sidekiq/version.rb +1 -1
  42. data/lib/sidekiq/web.rb +4 -4
  43. data/lib/sidekiq/web/action.rb +2 -6
  44. data/lib/sidekiq/web/application.rb +33 -16
  45. data/lib/sidekiq/web/helpers.rb +69 -22
  46. data/lib/sidekiq/web/router.rb +10 -10
  47. data/lib/sidekiq/worker.rb +118 -19
  48. data/sidekiq.gemspec +6 -17
  49. data/web/assets/javascripts/application.js +0 -0
  50. data/web/assets/javascripts/dashboard.js +32 -17
  51. data/web/assets/stylesheets/application-rtl.css +246 -0
  52. data/web/assets/stylesheets/application.css +371 -6
  53. data/web/assets/stylesheets/bootstrap-rtl.min.css +9 -0
  54. data/web/assets/stylesheets/bootstrap.css +2 -2
  55. data/web/locales/ar.yml +81 -0
  56. data/web/locales/en.yml +2 -0
  57. data/web/locales/es.yml +4 -3
  58. data/web/locales/fa.yml +1 -0
  59. data/web/locales/he.yml +79 -0
  60. data/web/locales/ja.yml +5 -3
  61. data/web/locales/ur.yml +80 -0
  62. data/web/views/_footer.erb +5 -2
  63. data/web/views/_nav.erb +4 -18
  64. data/web/views/_paging.erb +1 -1
  65. data/web/views/busy.erb +9 -5
  66. data/web/views/dashboard.erb +1 -1
  67. data/web/views/layout.erb +11 -2
  68. data/web/views/morgue.erb +4 -4
  69. data/web/views/queue.erb +8 -7
  70. data/web/views/queues.erb +2 -0
  71. data/web/views/retries.erb +9 -5
  72. data/web/views/scheduled.erb +2 -2
  73. metadata +31 -160
  74. data/lib/sidekiq/middleware/server/logging.rb +0 -31
  75. data/lib/sidekiq/middleware/server/retry_jobs.rb +0 -205
@@ -1,119 +1 @@
1
- # frozen_string_literal: true
2
- begin
3
- require 'active_support/core_ext/class/attribute'
4
- rescue LoadError
5
-
6
- # A dumbed down version of ActiveSupport's
7
- # Class#class_attribute helper.
8
- class Class
9
- def class_attribute(*attrs)
10
- instance_writer = true
11
-
12
- attrs.each do |name|
13
- class_eval <<-RUBY, __FILE__, __LINE__ + 1
14
- def self.#{name}() nil end
15
- def self.#{name}?() !!#{name} end
16
-
17
- def self.#{name}=(val)
18
- singleton_class.class_eval do
19
- define_method(:#{name}) { val }
20
- end
21
-
22
- if singleton_class?
23
- class_eval do
24
- def #{name}
25
- defined?(@#{name}) ? @#{name} : singleton_class.#{name}
26
- end
27
- end
28
- end
29
- val
30
- end
31
-
32
- def #{name}
33
- defined?(@#{name}) ? @#{name} : self.class.#{name}
34
- end
35
-
36
- def #{name}?
37
- !!#{name}
38
- end
39
- RUBY
40
-
41
- attr_writer name if instance_writer
42
- end
43
- end
44
-
45
- private
46
- def singleton_class?
47
- ancestors.first != self
48
- end
49
- end
50
- end
51
-
52
- begin
53
- require 'active_support/core_ext/hash/keys'
54
- require 'active_support/core_ext/hash/deep_merge'
55
- rescue LoadError
56
- class Hash
57
- def stringify_keys
58
- keys.each do |key|
59
- self[key.to_s] = delete(key)
60
- end
61
- self
62
- end if !{}.respond_to?(:stringify_keys)
63
-
64
- def symbolize_keys
65
- keys.each do |key|
66
- self[(key.to_sym rescue key) || key] = delete(key)
67
- end
68
- self
69
- end if !{}.respond_to?(:symbolize_keys)
70
-
71
- def deep_merge(other_hash, &block)
72
- dup.deep_merge!(other_hash, &block)
73
- end if !{}.respond_to?(:deep_merge)
74
-
75
- def deep_merge!(other_hash, &block)
76
- other_hash.each_pair do |k,v|
77
- tv = self[k]
78
- if tv.is_a?(Hash) && v.is_a?(Hash)
79
- self[k] = tv.deep_merge(v, &block)
80
- else
81
- self[k] = block && tv ? block.call(k, tv, v) : v
82
- end
83
- end
84
- self
85
- end if !{}.respond_to?(:deep_merge!)
86
- end
87
- end
88
-
89
- begin
90
- require 'active_support/core_ext/string/inflections'
91
- rescue LoadError
92
- class String
93
- def constantize
94
- names = self.split('::')
95
- names.shift if names.empty? || names.first.empty?
96
-
97
- constant = Object
98
- names.each do |name|
99
- constant = constant.const_defined?(name) ? constant.const_get(name) : constant.const_missing(name)
100
- end
101
- constant
102
- end
103
- end if !"".respond_to?(:constantize)
104
- end
105
-
106
-
107
- begin
108
- require 'active_support/core_ext/kernel/reporting'
109
- rescue LoadError
110
- module Kernel
111
- module_function
112
- def silence_warnings
113
- old_verbose, $VERBOSE = $VERBOSE, nil
114
- yield
115
- ensure
116
- $VERBOSE = old_verbose
117
- end
118
- end
119
- end
1
+ raise "no longer used, will be removed in 5.1"
@@ -0,0 +1,221 @@
1
+ #!/usr/bin/env ruby
2
+
3
+ require 'fileutils'
4
+ require 'sidekiq/api'
5
+
6
+ class Sidekiq::Ctl
7
+ DEFAULT_KILL_TIMEOUT = 10
8
+ CMD = File.basename($0)
9
+
10
+ attr_reader :stage, :pidfile, :kill_timeout
11
+
12
+ def self.print_usage
13
+ puts "#{CMD} - control Sidekiq from the command line."
14
+ puts
15
+ puts "Usage: #{CMD} quiet <pidfile> <kill_timeout>"
16
+ puts " #{CMD} stop <pidfile> <kill_timeout>"
17
+ puts " #{CMD} status <section>"
18
+ puts
19
+ puts " <pidfile> is path to a pidfile"
20
+ puts " <kill_timeout> is number of seconds to wait until Sidekiq exits"
21
+ puts " (default: #{Sidekiqctl::DEFAULT_KILL_TIMEOUT}), after which Sidekiq will be KILL'd"
22
+ puts
23
+ puts " <section> (optional) view a specific section of the status output"
24
+ puts " Valid sections are: #{Sidekiqctl::Status::VALID_SECTIONS.join(', ')}"
25
+ puts
26
+ puts "Be sure to set the kill_timeout LONGER than Sidekiq's -t timeout. If you want"
27
+ puts "to wait 60 seconds for jobs to finish, use `sidekiq -t 60` and `sidekiqctl stop"
28
+ puts " path_to_pidfile 61`"
29
+ puts
30
+ end
31
+
32
+ def initialize(stage, pidfile, timeout)
33
+ @stage = stage
34
+ @pidfile = pidfile
35
+ @kill_timeout = timeout
36
+
37
+ done('No pidfile given', :error) if !pidfile
38
+ done("Pidfile #{pidfile} does not exist", :warn) if !File.exist?(pidfile)
39
+ done('Invalid pidfile content', :error) if pid == 0
40
+
41
+ fetch_process
42
+
43
+ begin
44
+ send(stage)
45
+ rescue NoMethodError
46
+ done "Invalid command: #{stage}", :error
47
+ end
48
+ end
49
+
50
+ def fetch_process
51
+ Process.kill(0, pid)
52
+ rescue Errno::ESRCH
53
+ done "Process doesn't exist", :error
54
+ # We were not allowed to send a signal, but the process must have existed
55
+ # when Process.kill() was called.
56
+ rescue Errno::EPERM
57
+ return pid
58
+ end
59
+
60
+ def done(msg, error = nil)
61
+ puts msg
62
+ exit(exit_signal(error))
63
+ end
64
+
65
+ def exit_signal(error)
66
+ (error == :error) ? 1 : 0
67
+ end
68
+
69
+ def pid
70
+ @pid ||= File.read(pidfile).to_i
71
+ end
72
+
73
+ def quiet
74
+ `kill -TSTP #{pid}`
75
+ end
76
+
77
+ def stop
78
+ `kill -TERM #{pid}`
79
+ kill_timeout.times do
80
+ begin
81
+ Process.kill(0, pid)
82
+ rescue Errno::ESRCH
83
+ FileUtils.rm_f pidfile
84
+ done 'Sidekiq shut down gracefully.'
85
+ rescue Errno::EPERM
86
+ done 'Not permitted to shut down Sidekiq.'
87
+ end
88
+ sleep 1
89
+ end
90
+ `kill -9 #{pid}`
91
+ FileUtils.rm_f pidfile
92
+ done 'Sidekiq shut down forcefully.'
93
+ end
94
+ alias_method :shutdown, :stop
95
+
96
+ class Status
97
+ VALID_SECTIONS = %w[all version overview processes queues]
98
+ def display(section = nil)
99
+ section ||= 'all'
100
+ unless VALID_SECTIONS.include? section
101
+ puts "I don't know how to check the status of '#{section}'!"
102
+ puts "Try one of these: #{VALID_SECTIONS.join(', ')}"
103
+ return
104
+ end
105
+ send(section)
106
+ rescue StandardError => e
107
+ puts "Couldn't get status: #{e}"
108
+ end
109
+
110
+ def all
111
+ version
112
+ puts
113
+ overview
114
+ puts
115
+ processes
116
+ puts
117
+ queues
118
+ end
119
+
120
+ def version
121
+ puts "Sidekiq #{Sidekiq::VERSION}"
122
+ puts Time.now
123
+ end
124
+
125
+ def overview
126
+ puts '---- Overview ----'
127
+ puts " Processed: #{delimit stats.processed}"
128
+ puts " Failed: #{delimit stats.failed}"
129
+ puts " Busy: #{delimit stats.workers_size}"
130
+ puts " Enqueued: #{delimit stats.enqueued}"
131
+ puts " Retries: #{delimit stats.retry_size}"
132
+ puts " Scheduled: #{delimit stats.scheduled_size}"
133
+ puts " Dead: #{delimit stats.dead_size}"
134
+ end
135
+
136
+ def processes
137
+ puts "---- Processes (#{process_set.size}) ----"
138
+ process_set.each_with_index do |process, index|
139
+ puts "#{process['identity']} #{tags_for(process)}"
140
+ puts " Started: #{Time.at(process['started_at'])} (#{time_ago(process['started_at'])})"
141
+ puts " Threads: #{process['concurrency']} (#{process['busy']} busy)"
142
+ puts " Queues: #{split_multiline(process['queues'].sort, pad: 11)}"
143
+ puts '' unless (index+1) == process_set.size
144
+ end
145
+ end
146
+
147
+ COL_PAD = 2
148
+ def queues
149
+ puts "---- Queues (#{queue_data.size}) ----"
150
+ columns = {
151
+ name: [:ljust, (['name'] + queue_data.map(&:name)).map(&:length).max + COL_PAD],
152
+ size: [:rjust, (['size'] + queue_data.map(&:size)).map(&:length).max + COL_PAD],
153
+ latency: [:rjust, (['latency'] + queue_data.map(&:latency)).map(&:length).max + COL_PAD]
154
+ }
155
+ columns.each { |col, (dir, width)| print col.to_s.upcase.public_send(dir, width) }
156
+ puts
157
+ queue_data.each do |q|
158
+ columns.each do |col, (dir, width)|
159
+ print q.send(col).public_send(dir, width)
160
+ end
161
+ puts
162
+ end
163
+ end
164
+
165
+ private
166
+
167
+ def delimit(number)
168
+ number.to_s.reverse.scan(/.{1,3}/).join(',').reverse
169
+ end
170
+
171
+ def split_multiline(values, opts = {})
172
+ return 'none' unless values
173
+ pad = opts[:pad] || 0
174
+ max_length = opts[:max_length] || (80 - pad)
175
+ out = []
176
+ line = ''
177
+ values.each do |value|
178
+ if (line.length + value.length) > max_length
179
+ out << line
180
+ line = ' ' * pad
181
+ end
182
+ line << value + ', '
183
+ end
184
+ out << line[0..-3]
185
+ out.join("\n")
186
+ end
187
+
188
+ def tags_for(process)
189
+ tags = [
190
+ process['tag'],
191
+ process['labels'],
192
+ (process['quiet'] == 'true' ? 'quiet' : nil)
193
+ ].flatten.compact
194
+ tags.any? ? "[#{tags.join('] [')}]" : nil
195
+ end
196
+
197
+ def time_ago(timestamp)
198
+ seconds = Time.now - Time.at(timestamp)
199
+ return 'just now' if seconds < 60
200
+ return 'a minute ago' if seconds < 120
201
+ return "#{seconds.floor / 60} minutes ago" if seconds < 3600
202
+ return 'an hour ago' if seconds < 7200
203
+ "#{seconds.floor / 60 / 60} hours ago"
204
+ end
205
+
206
+ QUEUE_STRUCT = Struct.new(:name, :size, :latency)
207
+ def queue_data
208
+ @queue_data ||= Sidekiq::Queue.all.map do |q|
209
+ QUEUE_STRUCT.new(q.name, q.size.to_s, sprintf('%#.2f', q.latency))
210
+ end
211
+ end
212
+
213
+ def process_set
214
+ @process_set ||= Sidekiq::ProcessSet.new
215
+ end
216
+
217
+ def stats
218
+ @stats ||= Sidekiq::Stats.new
219
+ end
220
+ end
221
+ end
@@ -0,0 +1,42 @@
1
+ # frozen_string_literal: true
2
+ module Sidekiq
3
+ module Extensions
4
+
5
+ def self.enable_delay!
6
+ if defined?(::ActiveSupport)
7
+ require 'sidekiq/extensions/active_record'
8
+ require 'sidekiq/extensions/action_mailer'
9
+
10
+ # Need to patch Psych so it can autoload classes whose names are serialized
11
+ # in the delayed YAML.
12
+ Psych::Visitors::ToRuby.prepend(Sidekiq::Extensions::PsychAutoload)
13
+
14
+ ActiveSupport.on_load(:active_record) do
15
+ include Sidekiq::Extensions::ActiveRecord
16
+ end
17
+ ActiveSupport.on_load(:action_mailer) do
18
+ extend Sidekiq::Extensions::ActionMailer
19
+ end
20
+ end
21
+
22
+ require 'sidekiq/extensions/class_methods'
23
+ Module.__send__(:include, Sidekiq::Extensions::Klass)
24
+ end
25
+
26
+ module PsychAutoload
27
+ def resolve_class(klass_name)
28
+ return nil if !klass_name || klass_name.empty?
29
+ # constantize
30
+ names = klass_name.split('::')
31
+ names.shift if names.empty? || names.first.empty?
32
+
33
+ names.inject(Object) do |constant, name|
34
+ constant.const_defined?(name) ? constant.const_get(name) : constant.const_missing(name)
35
+ end
36
+ rescue NameError
37
+ super
38
+ end
39
+ end
40
+ end
41
+ end
42
+
@@ -7,11 +7,10 @@ module Sidekiq
7
7
  class Logger
8
8
  def call(ex, ctxHash)
9
9
  Sidekiq.logger.warn(Sidekiq.dump_json(ctxHash)) if !ctxHash.empty?
10
- Sidekiq.logger.warn "#{ex.class.name}: #{ex.message}"
11
- Sidekiq.logger.warn ex.backtrace.join("\n") unless ex.backtrace.nil?
10
+ Sidekiq.logger.warn("#{ex.class.name}: #{ex.message}")
11
+ Sidekiq.logger.warn(ex.backtrace.join("\n")) unless ex.backtrace.nil?
12
12
  end
13
13
 
14
- # Set up default handler which just logs the error
15
14
  Sidekiq.error_handlers << Sidekiq::ExceptionHandler::Logger.new
16
15
  end
17
16
 
@@ -26,6 +25,5 @@ module Sidekiq
26
25
  end
27
26
  end
28
27
  end
29
-
30
28
  end
31
29
  end
@@ -3,6 +3,8 @@ require 'yaml'
3
3
 
4
4
  module Sidekiq
5
5
  module Extensions
6
+ SIZE_LIMIT = 8_192
7
+
6
8
  class Proxy < BasicObject
7
9
  def initialize(performable, target, options={})
8
10
  @performable = performable
@@ -17,7 +19,11 @@ module Sidekiq
17
19
  # to JSON and then deserialized on the other side back into a
18
20
  # Ruby object.
19
21
  obj = [@target, name, args]
20
- @performable.client_push({ 'class' => @performable, 'args' => [::YAML.dump(obj)] }.merge(@opts))
22
+ marshalled = ::YAML.dump(obj)
23
+ if marshalled.size > SIZE_LIMIT
24
+ ::Sidekiq.logger.warn { "#{@target}.#{name} job argument is #{marshalled.bytesize} bytes, you should refactor it to reduce the size" }
25
+ end
26
+ @performable.client_push({ 'class' => @performable, 'args' => [marshalled] }.merge(@opts))
21
27
  end
22
28
  end
23
29
 
@@ -13,7 +13,7 @@ module Sidekiq
13
13
  end
14
14
 
15
15
  def queue_name
16
- queue.sub(/.*queue:/, ''.freeze)
16
+ queue.sub(/.*queue:/, '')
17
17
  end
18
18
 
19
19
  def requeue
@@ -0,0 +1,25 @@
1
+ # frozen_string_literal: true
2
+ module Sidekiq
3
+ class JobLogger
4
+
5
+ def call(item, queue)
6
+ start = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
7
+ logger.info("start")
8
+ yield
9
+ logger.info("done: #{elapsed(start)} sec")
10
+ rescue Exception
11
+ logger.info("fail: #{elapsed(start)} sec")
12
+ raise
13
+ end
14
+
15
+ private
16
+
17
+ def elapsed(start)
18
+ (::Process.clock_gettime(::Process::CLOCK_MONOTONIC) - start).round(3)
19
+ end
20
+
21
+ def logger
22
+ Sidekiq.logger
23
+ end
24
+ end
25
+ end