sidekiq 4.2.10 → 5.2.7
Sign up to get free protection for your applications and to get access to all the features.
Potentially problematic release.
This version of sidekiq might be problematic. Click here for more details.
- checksums.yaml +5 -5
- data/.circleci/config.yml +61 -0
- data/.github/issue_template.md +3 -1
- data/.gitignore +3 -0
- data/.travis.yml +6 -13
- data/5.0-Upgrade.md +56 -0
- data/COMM-LICENSE +12 -10
- data/Changes.md +158 -1
- data/Ent-Changes.md +67 -2
- data/Gemfile +14 -20
- data/LICENSE +1 -1
- data/Pro-4.0-Upgrade.md +35 -0
- data/Pro-Changes.md +133 -2
- data/README.md +8 -6
- data/Rakefile +2 -5
- data/bin/sidekiqctl +13 -92
- data/bin/sidekiqload +5 -10
- data/lib/generators/sidekiq/templates/worker_spec.rb.erb +1 -1
- data/lib/sidekiq.rb +27 -27
- data/lib/sidekiq/api.rb +145 -57
- data/lib/sidekiq/cli.rb +120 -81
- data/lib/sidekiq/client.rb +25 -18
- data/lib/sidekiq/core_ext.rb +1 -119
- data/lib/sidekiq/ctl.rb +221 -0
- data/lib/sidekiq/delay.rb +42 -0
- data/lib/sidekiq/exception_handler.rb +2 -4
- data/lib/sidekiq/extensions/generic_proxy.rb +7 -1
- data/lib/sidekiq/fetch.rb +1 -1
- data/lib/sidekiq/job_logger.rb +25 -0
- data/lib/sidekiq/job_retry.rb +262 -0
- data/lib/sidekiq/launcher.rb +19 -19
- data/lib/sidekiq/logging.rb +18 -2
- data/lib/sidekiq/manager.rb +5 -6
- data/lib/sidekiq/middleware/server/active_record.rb +10 -0
- data/lib/sidekiq/processor.rb +126 -48
- data/lib/sidekiq/rails.rb +8 -73
- data/lib/sidekiq/redis_connection.rb +43 -5
- data/lib/sidekiq/scheduled.rb +35 -8
- data/lib/sidekiq/testing.rb +16 -7
- data/lib/sidekiq/util.rb +5 -2
- data/lib/sidekiq/version.rb +1 -1
- data/lib/sidekiq/web.rb +4 -4
- data/lib/sidekiq/web/action.rb +2 -6
- data/lib/sidekiq/web/application.rb +33 -16
- data/lib/sidekiq/web/helpers.rb +69 -22
- data/lib/sidekiq/web/router.rb +10 -10
- data/lib/sidekiq/worker.rb +118 -19
- data/sidekiq.gemspec +6 -17
- data/web/assets/javascripts/application.js +0 -0
- data/web/assets/javascripts/dashboard.js +32 -17
- data/web/assets/stylesheets/application-rtl.css +246 -0
- data/web/assets/stylesheets/application.css +371 -6
- data/web/assets/stylesheets/bootstrap-rtl.min.css +9 -0
- data/web/assets/stylesheets/bootstrap.css +2 -2
- data/web/locales/ar.yml +81 -0
- data/web/locales/en.yml +2 -0
- data/web/locales/es.yml +4 -3
- data/web/locales/fa.yml +1 -0
- data/web/locales/he.yml +79 -0
- data/web/locales/ja.yml +5 -3
- data/web/locales/ur.yml +80 -0
- data/web/views/_footer.erb +5 -2
- data/web/views/_nav.erb +4 -18
- data/web/views/_paging.erb +1 -1
- data/web/views/busy.erb +9 -5
- data/web/views/dashboard.erb +1 -1
- data/web/views/layout.erb +11 -2
- data/web/views/morgue.erb +4 -4
- data/web/views/queue.erb +8 -7
- data/web/views/queues.erb +2 -0
- data/web/views/retries.erb +9 -5
- data/web/views/scheduled.erb +2 -2
- metadata +31 -160
- data/lib/sidekiq/middleware/server/logging.rb +0 -31
- data/lib/sidekiq/middleware/server/retry_jobs.rb +0 -205
data/lib/sidekiq/core_ext.rb
CHANGED
@@ -1,119 +1 @@
|
|
1
|
-
|
2
|
-
begin
|
3
|
-
require 'active_support/core_ext/class/attribute'
|
4
|
-
rescue LoadError
|
5
|
-
|
6
|
-
# A dumbed down version of ActiveSupport's
|
7
|
-
# Class#class_attribute helper.
|
8
|
-
class Class
|
9
|
-
def class_attribute(*attrs)
|
10
|
-
instance_writer = true
|
11
|
-
|
12
|
-
attrs.each do |name|
|
13
|
-
class_eval <<-RUBY, __FILE__, __LINE__ + 1
|
14
|
-
def self.#{name}() nil end
|
15
|
-
def self.#{name}?() !!#{name} end
|
16
|
-
|
17
|
-
def self.#{name}=(val)
|
18
|
-
singleton_class.class_eval do
|
19
|
-
define_method(:#{name}) { val }
|
20
|
-
end
|
21
|
-
|
22
|
-
if singleton_class?
|
23
|
-
class_eval do
|
24
|
-
def #{name}
|
25
|
-
defined?(@#{name}) ? @#{name} : singleton_class.#{name}
|
26
|
-
end
|
27
|
-
end
|
28
|
-
end
|
29
|
-
val
|
30
|
-
end
|
31
|
-
|
32
|
-
def #{name}
|
33
|
-
defined?(@#{name}) ? @#{name} : self.class.#{name}
|
34
|
-
end
|
35
|
-
|
36
|
-
def #{name}?
|
37
|
-
!!#{name}
|
38
|
-
end
|
39
|
-
RUBY
|
40
|
-
|
41
|
-
attr_writer name if instance_writer
|
42
|
-
end
|
43
|
-
end
|
44
|
-
|
45
|
-
private
|
46
|
-
def singleton_class?
|
47
|
-
ancestors.first != self
|
48
|
-
end
|
49
|
-
end
|
50
|
-
end
|
51
|
-
|
52
|
-
begin
|
53
|
-
require 'active_support/core_ext/hash/keys'
|
54
|
-
require 'active_support/core_ext/hash/deep_merge'
|
55
|
-
rescue LoadError
|
56
|
-
class Hash
|
57
|
-
def stringify_keys
|
58
|
-
keys.each do |key|
|
59
|
-
self[key.to_s] = delete(key)
|
60
|
-
end
|
61
|
-
self
|
62
|
-
end if !{}.respond_to?(:stringify_keys)
|
63
|
-
|
64
|
-
def symbolize_keys
|
65
|
-
keys.each do |key|
|
66
|
-
self[(key.to_sym rescue key) || key] = delete(key)
|
67
|
-
end
|
68
|
-
self
|
69
|
-
end if !{}.respond_to?(:symbolize_keys)
|
70
|
-
|
71
|
-
def deep_merge(other_hash, &block)
|
72
|
-
dup.deep_merge!(other_hash, &block)
|
73
|
-
end if !{}.respond_to?(:deep_merge)
|
74
|
-
|
75
|
-
def deep_merge!(other_hash, &block)
|
76
|
-
other_hash.each_pair do |k,v|
|
77
|
-
tv = self[k]
|
78
|
-
if tv.is_a?(Hash) && v.is_a?(Hash)
|
79
|
-
self[k] = tv.deep_merge(v, &block)
|
80
|
-
else
|
81
|
-
self[k] = block && tv ? block.call(k, tv, v) : v
|
82
|
-
end
|
83
|
-
end
|
84
|
-
self
|
85
|
-
end if !{}.respond_to?(:deep_merge!)
|
86
|
-
end
|
87
|
-
end
|
88
|
-
|
89
|
-
begin
|
90
|
-
require 'active_support/core_ext/string/inflections'
|
91
|
-
rescue LoadError
|
92
|
-
class String
|
93
|
-
def constantize
|
94
|
-
names = self.split('::')
|
95
|
-
names.shift if names.empty? || names.first.empty?
|
96
|
-
|
97
|
-
constant = Object
|
98
|
-
names.each do |name|
|
99
|
-
constant = constant.const_defined?(name) ? constant.const_get(name) : constant.const_missing(name)
|
100
|
-
end
|
101
|
-
constant
|
102
|
-
end
|
103
|
-
end if !"".respond_to?(:constantize)
|
104
|
-
end
|
105
|
-
|
106
|
-
|
107
|
-
begin
|
108
|
-
require 'active_support/core_ext/kernel/reporting'
|
109
|
-
rescue LoadError
|
110
|
-
module Kernel
|
111
|
-
module_function
|
112
|
-
def silence_warnings
|
113
|
-
old_verbose, $VERBOSE = $VERBOSE, nil
|
114
|
-
yield
|
115
|
-
ensure
|
116
|
-
$VERBOSE = old_verbose
|
117
|
-
end
|
118
|
-
end
|
119
|
-
end
|
1
|
+
raise "no longer used, will be removed in 5.1"
|
data/lib/sidekiq/ctl.rb
ADDED
@@ -0,0 +1,221 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
|
3
|
+
require 'fileutils'
|
4
|
+
require 'sidekiq/api'
|
5
|
+
|
6
|
+
class Sidekiq::Ctl
|
7
|
+
DEFAULT_KILL_TIMEOUT = 10
|
8
|
+
CMD = File.basename($0)
|
9
|
+
|
10
|
+
attr_reader :stage, :pidfile, :kill_timeout
|
11
|
+
|
12
|
+
def self.print_usage
|
13
|
+
puts "#{CMD} - control Sidekiq from the command line."
|
14
|
+
puts
|
15
|
+
puts "Usage: #{CMD} quiet <pidfile> <kill_timeout>"
|
16
|
+
puts " #{CMD} stop <pidfile> <kill_timeout>"
|
17
|
+
puts " #{CMD} status <section>"
|
18
|
+
puts
|
19
|
+
puts " <pidfile> is path to a pidfile"
|
20
|
+
puts " <kill_timeout> is number of seconds to wait until Sidekiq exits"
|
21
|
+
puts " (default: #{Sidekiqctl::DEFAULT_KILL_TIMEOUT}), after which Sidekiq will be KILL'd"
|
22
|
+
puts
|
23
|
+
puts " <section> (optional) view a specific section of the status output"
|
24
|
+
puts " Valid sections are: #{Sidekiqctl::Status::VALID_SECTIONS.join(', ')}"
|
25
|
+
puts
|
26
|
+
puts "Be sure to set the kill_timeout LONGER than Sidekiq's -t timeout. If you want"
|
27
|
+
puts "to wait 60 seconds for jobs to finish, use `sidekiq -t 60` and `sidekiqctl stop"
|
28
|
+
puts " path_to_pidfile 61`"
|
29
|
+
puts
|
30
|
+
end
|
31
|
+
|
32
|
+
def initialize(stage, pidfile, timeout)
|
33
|
+
@stage = stage
|
34
|
+
@pidfile = pidfile
|
35
|
+
@kill_timeout = timeout
|
36
|
+
|
37
|
+
done('No pidfile given', :error) if !pidfile
|
38
|
+
done("Pidfile #{pidfile} does not exist", :warn) if !File.exist?(pidfile)
|
39
|
+
done('Invalid pidfile content', :error) if pid == 0
|
40
|
+
|
41
|
+
fetch_process
|
42
|
+
|
43
|
+
begin
|
44
|
+
send(stage)
|
45
|
+
rescue NoMethodError
|
46
|
+
done "Invalid command: #{stage}", :error
|
47
|
+
end
|
48
|
+
end
|
49
|
+
|
50
|
+
def fetch_process
|
51
|
+
Process.kill(0, pid)
|
52
|
+
rescue Errno::ESRCH
|
53
|
+
done "Process doesn't exist", :error
|
54
|
+
# We were not allowed to send a signal, but the process must have existed
|
55
|
+
# when Process.kill() was called.
|
56
|
+
rescue Errno::EPERM
|
57
|
+
return pid
|
58
|
+
end
|
59
|
+
|
60
|
+
def done(msg, error = nil)
|
61
|
+
puts msg
|
62
|
+
exit(exit_signal(error))
|
63
|
+
end
|
64
|
+
|
65
|
+
def exit_signal(error)
|
66
|
+
(error == :error) ? 1 : 0
|
67
|
+
end
|
68
|
+
|
69
|
+
def pid
|
70
|
+
@pid ||= File.read(pidfile).to_i
|
71
|
+
end
|
72
|
+
|
73
|
+
def quiet
|
74
|
+
`kill -TSTP #{pid}`
|
75
|
+
end
|
76
|
+
|
77
|
+
def stop
|
78
|
+
`kill -TERM #{pid}`
|
79
|
+
kill_timeout.times do
|
80
|
+
begin
|
81
|
+
Process.kill(0, pid)
|
82
|
+
rescue Errno::ESRCH
|
83
|
+
FileUtils.rm_f pidfile
|
84
|
+
done 'Sidekiq shut down gracefully.'
|
85
|
+
rescue Errno::EPERM
|
86
|
+
done 'Not permitted to shut down Sidekiq.'
|
87
|
+
end
|
88
|
+
sleep 1
|
89
|
+
end
|
90
|
+
`kill -9 #{pid}`
|
91
|
+
FileUtils.rm_f pidfile
|
92
|
+
done 'Sidekiq shut down forcefully.'
|
93
|
+
end
|
94
|
+
alias_method :shutdown, :stop
|
95
|
+
|
96
|
+
class Status
|
97
|
+
VALID_SECTIONS = %w[all version overview processes queues]
|
98
|
+
def display(section = nil)
|
99
|
+
section ||= 'all'
|
100
|
+
unless VALID_SECTIONS.include? section
|
101
|
+
puts "I don't know how to check the status of '#{section}'!"
|
102
|
+
puts "Try one of these: #{VALID_SECTIONS.join(', ')}"
|
103
|
+
return
|
104
|
+
end
|
105
|
+
send(section)
|
106
|
+
rescue StandardError => e
|
107
|
+
puts "Couldn't get status: #{e}"
|
108
|
+
end
|
109
|
+
|
110
|
+
def all
|
111
|
+
version
|
112
|
+
puts
|
113
|
+
overview
|
114
|
+
puts
|
115
|
+
processes
|
116
|
+
puts
|
117
|
+
queues
|
118
|
+
end
|
119
|
+
|
120
|
+
def version
|
121
|
+
puts "Sidekiq #{Sidekiq::VERSION}"
|
122
|
+
puts Time.now
|
123
|
+
end
|
124
|
+
|
125
|
+
def overview
|
126
|
+
puts '---- Overview ----'
|
127
|
+
puts " Processed: #{delimit stats.processed}"
|
128
|
+
puts " Failed: #{delimit stats.failed}"
|
129
|
+
puts " Busy: #{delimit stats.workers_size}"
|
130
|
+
puts " Enqueued: #{delimit stats.enqueued}"
|
131
|
+
puts " Retries: #{delimit stats.retry_size}"
|
132
|
+
puts " Scheduled: #{delimit stats.scheduled_size}"
|
133
|
+
puts " Dead: #{delimit stats.dead_size}"
|
134
|
+
end
|
135
|
+
|
136
|
+
def processes
|
137
|
+
puts "---- Processes (#{process_set.size}) ----"
|
138
|
+
process_set.each_with_index do |process, index|
|
139
|
+
puts "#{process['identity']} #{tags_for(process)}"
|
140
|
+
puts " Started: #{Time.at(process['started_at'])} (#{time_ago(process['started_at'])})"
|
141
|
+
puts " Threads: #{process['concurrency']} (#{process['busy']} busy)"
|
142
|
+
puts " Queues: #{split_multiline(process['queues'].sort, pad: 11)}"
|
143
|
+
puts '' unless (index+1) == process_set.size
|
144
|
+
end
|
145
|
+
end
|
146
|
+
|
147
|
+
COL_PAD = 2
|
148
|
+
def queues
|
149
|
+
puts "---- Queues (#{queue_data.size}) ----"
|
150
|
+
columns = {
|
151
|
+
name: [:ljust, (['name'] + queue_data.map(&:name)).map(&:length).max + COL_PAD],
|
152
|
+
size: [:rjust, (['size'] + queue_data.map(&:size)).map(&:length).max + COL_PAD],
|
153
|
+
latency: [:rjust, (['latency'] + queue_data.map(&:latency)).map(&:length).max + COL_PAD]
|
154
|
+
}
|
155
|
+
columns.each { |col, (dir, width)| print col.to_s.upcase.public_send(dir, width) }
|
156
|
+
puts
|
157
|
+
queue_data.each do |q|
|
158
|
+
columns.each do |col, (dir, width)|
|
159
|
+
print q.send(col).public_send(dir, width)
|
160
|
+
end
|
161
|
+
puts
|
162
|
+
end
|
163
|
+
end
|
164
|
+
|
165
|
+
private
|
166
|
+
|
167
|
+
def delimit(number)
|
168
|
+
number.to_s.reverse.scan(/.{1,3}/).join(',').reverse
|
169
|
+
end
|
170
|
+
|
171
|
+
def split_multiline(values, opts = {})
|
172
|
+
return 'none' unless values
|
173
|
+
pad = opts[:pad] || 0
|
174
|
+
max_length = opts[:max_length] || (80 - pad)
|
175
|
+
out = []
|
176
|
+
line = ''
|
177
|
+
values.each do |value|
|
178
|
+
if (line.length + value.length) > max_length
|
179
|
+
out << line
|
180
|
+
line = ' ' * pad
|
181
|
+
end
|
182
|
+
line << value + ', '
|
183
|
+
end
|
184
|
+
out << line[0..-3]
|
185
|
+
out.join("\n")
|
186
|
+
end
|
187
|
+
|
188
|
+
def tags_for(process)
|
189
|
+
tags = [
|
190
|
+
process['tag'],
|
191
|
+
process['labels'],
|
192
|
+
(process['quiet'] == 'true' ? 'quiet' : nil)
|
193
|
+
].flatten.compact
|
194
|
+
tags.any? ? "[#{tags.join('] [')}]" : nil
|
195
|
+
end
|
196
|
+
|
197
|
+
def time_ago(timestamp)
|
198
|
+
seconds = Time.now - Time.at(timestamp)
|
199
|
+
return 'just now' if seconds < 60
|
200
|
+
return 'a minute ago' if seconds < 120
|
201
|
+
return "#{seconds.floor / 60} minutes ago" if seconds < 3600
|
202
|
+
return 'an hour ago' if seconds < 7200
|
203
|
+
"#{seconds.floor / 60 / 60} hours ago"
|
204
|
+
end
|
205
|
+
|
206
|
+
QUEUE_STRUCT = Struct.new(:name, :size, :latency)
|
207
|
+
def queue_data
|
208
|
+
@queue_data ||= Sidekiq::Queue.all.map do |q|
|
209
|
+
QUEUE_STRUCT.new(q.name, q.size.to_s, sprintf('%#.2f', q.latency))
|
210
|
+
end
|
211
|
+
end
|
212
|
+
|
213
|
+
def process_set
|
214
|
+
@process_set ||= Sidekiq::ProcessSet.new
|
215
|
+
end
|
216
|
+
|
217
|
+
def stats
|
218
|
+
@stats ||= Sidekiq::Stats.new
|
219
|
+
end
|
220
|
+
end
|
221
|
+
end
|
@@ -0,0 +1,42 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
module Sidekiq
|
3
|
+
module Extensions
|
4
|
+
|
5
|
+
def self.enable_delay!
|
6
|
+
if defined?(::ActiveSupport)
|
7
|
+
require 'sidekiq/extensions/active_record'
|
8
|
+
require 'sidekiq/extensions/action_mailer'
|
9
|
+
|
10
|
+
# Need to patch Psych so it can autoload classes whose names are serialized
|
11
|
+
# in the delayed YAML.
|
12
|
+
Psych::Visitors::ToRuby.prepend(Sidekiq::Extensions::PsychAutoload)
|
13
|
+
|
14
|
+
ActiveSupport.on_load(:active_record) do
|
15
|
+
include Sidekiq::Extensions::ActiveRecord
|
16
|
+
end
|
17
|
+
ActiveSupport.on_load(:action_mailer) do
|
18
|
+
extend Sidekiq::Extensions::ActionMailer
|
19
|
+
end
|
20
|
+
end
|
21
|
+
|
22
|
+
require 'sidekiq/extensions/class_methods'
|
23
|
+
Module.__send__(:include, Sidekiq::Extensions::Klass)
|
24
|
+
end
|
25
|
+
|
26
|
+
module PsychAutoload
|
27
|
+
def resolve_class(klass_name)
|
28
|
+
return nil if !klass_name || klass_name.empty?
|
29
|
+
# constantize
|
30
|
+
names = klass_name.split('::')
|
31
|
+
names.shift if names.empty? || names.first.empty?
|
32
|
+
|
33
|
+
names.inject(Object) do |constant, name|
|
34
|
+
constant.const_defined?(name) ? constant.const_get(name) : constant.const_missing(name)
|
35
|
+
end
|
36
|
+
rescue NameError
|
37
|
+
super
|
38
|
+
end
|
39
|
+
end
|
40
|
+
end
|
41
|
+
end
|
42
|
+
|
@@ -7,11 +7,10 @@ module Sidekiq
|
|
7
7
|
class Logger
|
8
8
|
def call(ex, ctxHash)
|
9
9
|
Sidekiq.logger.warn(Sidekiq.dump_json(ctxHash)) if !ctxHash.empty?
|
10
|
-
Sidekiq.logger.warn
|
11
|
-
Sidekiq.logger.warn
|
10
|
+
Sidekiq.logger.warn("#{ex.class.name}: #{ex.message}")
|
11
|
+
Sidekiq.logger.warn(ex.backtrace.join("\n")) unless ex.backtrace.nil?
|
12
12
|
end
|
13
13
|
|
14
|
-
# Set up default handler which just logs the error
|
15
14
|
Sidekiq.error_handlers << Sidekiq::ExceptionHandler::Logger.new
|
16
15
|
end
|
17
16
|
|
@@ -26,6 +25,5 @@ module Sidekiq
|
|
26
25
|
end
|
27
26
|
end
|
28
27
|
end
|
29
|
-
|
30
28
|
end
|
31
29
|
end
|
@@ -3,6 +3,8 @@ require 'yaml'
|
|
3
3
|
|
4
4
|
module Sidekiq
|
5
5
|
module Extensions
|
6
|
+
SIZE_LIMIT = 8_192
|
7
|
+
|
6
8
|
class Proxy < BasicObject
|
7
9
|
def initialize(performable, target, options={})
|
8
10
|
@performable = performable
|
@@ -17,7 +19,11 @@ module Sidekiq
|
|
17
19
|
# to JSON and then deserialized on the other side back into a
|
18
20
|
# Ruby object.
|
19
21
|
obj = [@target, name, args]
|
20
|
-
|
22
|
+
marshalled = ::YAML.dump(obj)
|
23
|
+
if marshalled.size > SIZE_LIMIT
|
24
|
+
::Sidekiq.logger.warn { "#{@target}.#{name} job argument is #{marshalled.bytesize} bytes, you should refactor it to reduce the size" }
|
25
|
+
end
|
26
|
+
@performable.client_push({ 'class' => @performable, 'args' => [marshalled] }.merge(@opts))
|
21
27
|
end
|
22
28
|
end
|
23
29
|
|
data/lib/sidekiq/fetch.rb
CHANGED
@@ -0,0 +1,25 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
module Sidekiq
|
3
|
+
class JobLogger
|
4
|
+
|
5
|
+
def call(item, queue)
|
6
|
+
start = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
|
7
|
+
logger.info("start")
|
8
|
+
yield
|
9
|
+
logger.info("done: #{elapsed(start)} sec")
|
10
|
+
rescue Exception
|
11
|
+
logger.info("fail: #{elapsed(start)} sec")
|
12
|
+
raise
|
13
|
+
end
|
14
|
+
|
15
|
+
private
|
16
|
+
|
17
|
+
def elapsed(start)
|
18
|
+
(::Process.clock_gettime(::Process::CLOCK_MONOTONIC) - start).round(3)
|
19
|
+
end
|
20
|
+
|
21
|
+
def logger
|
22
|
+
Sidekiq.logger
|
23
|
+
end
|
24
|
+
end
|
25
|
+
end
|