sidekiq 5.2.7 → 6.0.3
Sign up to get free protection for your applications and to get access to all the features.
Potentially problematic release.
This version of sidekiq might be problematic. Click here for more details.
- checksums.yaml +4 -4
- data/.circleci/config.yml +21 -0
- data/.gitignore +0 -2
- data/.standard.yml +20 -0
- data/6.0-Upgrade.md +72 -0
- data/Changes.md +121 -0
- data/Ent-2.0-Upgrade.md +37 -0
- data/Ent-Changes.md +18 -0
- data/Gemfile +12 -11
- data/Gemfile.lock +196 -0
- data/Pro-5.0-Upgrade.md +25 -0
- data/Pro-Changes.md +18 -1
- data/README.md +18 -30
- data/Rakefile +5 -4
- data/bin/sidekiqload +32 -24
- data/bin/sidekiqmon +8 -0
- data/lib/generators/sidekiq/templates/worker_test.rb.erb +1 -1
- data/lib/generators/sidekiq/worker_generator.rb +20 -12
- data/lib/sidekiq.rb +61 -42
- data/lib/sidekiq/api.rb +230 -214
- data/lib/sidekiq/cli.rb +111 -174
- data/lib/sidekiq/client.rb +55 -46
- data/lib/sidekiq/delay.rb +5 -6
- data/lib/sidekiq/exception_handler.rb +10 -12
- data/lib/sidekiq/extensions/action_mailer.rb +10 -20
- data/lib/sidekiq/extensions/active_record.rb +9 -7
- data/lib/sidekiq/extensions/class_methods.rb +9 -7
- data/lib/sidekiq/extensions/generic_proxy.rb +4 -4
- data/lib/sidekiq/fetch.rb +11 -12
- data/lib/sidekiq/job_logger.rb +45 -7
- data/lib/sidekiq/job_retry.rb +60 -60
- data/lib/sidekiq/launcher.rb +57 -51
- data/lib/sidekiq/logger.rb +165 -0
- data/lib/sidekiq/manager.rb +7 -9
- data/lib/sidekiq/middleware/chain.rb +14 -4
- data/lib/sidekiq/middleware/i18n.rb +5 -7
- data/lib/sidekiq/monitor.rb +133 -0
- data/lib/sidekiq/paginator.rb +18 -14
- data/lib/sidekiq/processor.rb +67 -66
- data/lib/sidekiq/rails.rb +23 -29
- data/lib/sidekiq/redis_connection.rb +31 -37
- data/lib/sidekiq/scheduled.rb +28 -29
- data/lib/sidekiq/testing.rb +34 -23
- data/lib/sidekiq/testing/inline.rb +2 -1
- data/lib/sidekiq/util.rb +17 -16
- data/lib/sidekiq/version.rb +2 -1
- data/lib/sidekiq/web.rb +41 -49
- data/lib/sidekiq/web/action.rb +14 -10
- data/lib/sidekiq/web/application.rb +64 -66
- data/lib/sidekiq/web/helpers.rb +83 -72
- data/lib/sidekiq/web/router.rb +17 -14
- data/lib/sidekiq/worker.rb +124 -97
- data/sidekiq.gemspec +16 -16
- data/web/assets/javascripts/dashboard.js +4 -23
- data/web/assets/stylesheets/application-dark.css +125 -0
- data/web/assets/stylesheets/application.css +9 -0
- data/web/locales/de.yml +14 -2
- data/web/locales/ja.yml +2 -1
- data/web/views/_job_info.erb +2 -1
- data/web/views/busy.erb +4 -1
- data/web/views/dead.erb +2 -2
- data/web/views/layout.erb +1 -0
- data/web/views/morgue.erb +4 -1
- data/web/views/queue.erb +10 -1
- data/web/views/retries.erb +4 -1
- data/web/views/retry.erb +2 -2
- data/web/views/scheduled.erb +4 -1
- metadata +20 -29
- data/.travis.yml +0 -11
- data/bin/sidekiqctl +0 -20
- data/lib/sidekiq/core_ext.rb +0 -1
- data/lib/sidekiq/ctl.rb +0 -221
- data/lib/sidekiq/logging.rb +0 -122
- data/lib/sidekiq/middleware/server/active_record.rb +0 -23
data/.travis.yml
DELETED
data/bin/sidekiqctl
DELETED
@@ -1,20 +0,0 @@
|
|
1
|
-
#!/usr/bin/env ruby
|
2
|
-
|
3
|
-
require 'fileutils'
|
4
|
-
require 'sidekiq/api'
|
5
|
-
require 'sidekiq/ctl'
|
6
|
-
|
7
|
-
if ARGV[0] == 'status'
|
8
|
-
Sidekiq::Ctl::Status.new.display(ARGV[1])
|
9
|
-
else
|
10
|
-
if ARGV.length < 2
|
11
|
-
Sidekiq::Ctl.print_usage
|
12
|
-
else
|
13
|
-
stage = ARGV[0]
|
14
|
-
pidfile = ARGV[1]
|
15
|
-
timeout = ARGV[2].to_i
|
16
|
-
timeout = Sidekiq::Ctl::DEFAULT_KILL_TIMEOUT if timeout == 0
|
17
|
-
|
18
|
-
Sidekiq::Ctl.new(stage, pidfile, timeout)
|
19
|
-
end
|
20
|
-
end
|
data/lib/sidekiq/core_ext.rb
DELETED
@@ -1 +0,0 @@
|
|
1
|
-
raise "no longer used, will be removed in 5.1"
|
data/lib/sidekiq/ctl.rb
DELETED
@@ -1,221 +0,0 @@
|
|
1
|
-
#!/usr/bin/env ruby
|
2
|
-
|
3
|
-
require 'fileutils'
|
4
|
-
require 'sidekiq/api'
|
5
|
-
|
6
|
-
class Sidekiq::Ctl
|
7
|
-
DEFAULT_KILL_TIMEOUT = 10
|
8
|
-
CMD = File.basename($0)
|
9
|
-
|
10
|
-
attr_reader :stage, :pidfile, :kill_timeout
|
11
|
-
|
12
|
-
def self.print_usage
|
13
|
-
puts "#{CMD} - control Sidekiq from the command line."
|
14
|
-
puts
|
15
|
-
puts "Usage: #{CMD} quiet <pidfile> <kill_timeout>"
|
16
|
-
puts " #{CMD} stop <pidfile> <kill_timeout>"
|
17
|
-
puts " #{CMD} status <section>"
|
18
|
-
puts
|
19
|
-
puts " <pidfile> is path to a pidfile"
|
20
|
-
puts " <kill_timeout> is number of seconds to wait until Sidekiq exits"
|
21
|
-
puts " (default: #{Sidekiqctl::DEFAULT_KILL_TIMEOUT}), after which Sidekiq will be KILL'd"
|
22
|
-
puts
|
23
|
-
puts " <section> (optional) view a specific section of the status output"
|
24
|
-
puts " Valid sections are: #{Sidekiqctl::Status::VALID_SECTIONS.join(', ')}"
|
25
|
-
puts
|
26
|
-
puts "Be sure to set the kill_timeout LONGER than Sidekiq's -t timeout. If you want"
|
27
|
-
puts "to wait 60 seconds for jobs to finish, use `sidekiq -t 60` and `sidekiqctl stop"
|
28
|
-
puts " path_to_pidfile 61`"
|
29
|
-
puts
|
30
|
-
end
|
31
|
-
|
32
|
-
def initialize(stage, pidfile, timeout)
|
33
|
-
@stage = stage
|
34
|
-
@pidfile = pidfile
|
35
|
-
@kill_timeout = timeout
|
36
|
-
|
37
|
-
done('No pidfile given', :error) if !pidfile
|
38
|
-
done("Pidfile #{pidfile} does not exist", :warn) if !File.exist?(pidfile)
|
39
|
-
done('Invalid pidfile content', :error) if pid == 0
|
40
|
-
|
41
|
-
fetch_process
|
42
|
-
|
43
|
-
begin
|
44
|
-
send(stage)
|
45
|
-
rescue NoMethodError
|
46
|
-
done "Invalid command: #{stage}", :error
|
47
|
-
end
|
48
|
-
end
|
49
|
-
|
50
|
-
def fetch_process
|
51
|
-
Process.kill(0, pid)
|
52
|
-
rescue Errno::ESRCH
|
53
|
-
done "Process doesn't exist", :error
|
54
|
-
# We were not allowed to send a signal, but the process must have existed
|
55
|
-
# when Process.kill() was called.
|
56
|
-
rescue Errno::EPERM
|
57
|
-
return pid
|
58
|
-
end
|
59
|
-
|
60
|
-
def done(msg, error = nil)
|
61
|
-
puts msg
|
62
|
-
exit(exit_signal(error))
|
63
|
-
end
|
64
|
-
|
65
|
-
def exit_signal(error)
|
66
|
-
(error == :error) ? 1 : 0
|
67
|
-
end
|
68
|
-
|
69
|
-
def pid
|
70
|
-
@pid ||= File.read(pidfile).to_i
|
71
|
-
end
|
72
|
-
|
73
|
-
def quiet
|
74
|
-
`kill -TSTP #{pid}`
|
75
|
-
end
|
76
|
-
|
77
|
-
def stop
|
78
|
-
`kill -TERM #{pid}`
|
79
|
-
kill_timeout.times do
|
80
|
-
begin
|
81
|
-
Process.kill(0, pid)
|
82
|
-
rescue Errno::ESRCH
|
83
|
-
FileUtils.rm_f pidfile
|
84
|
-
done 'Sidekiq shut down gracefully.'
|
85
|
-
rescue Errno::EPERM
|
86
|
-
done 'Not permitted to shut down Sidekiq.'
|
87
|
-
end
|
88
|
-
sleep 1
|
89
|
-
end
|
90
|
-
`kill -9 #{pid}`
|
91
|
-
FileUtils.rm_f pidfile
|
92
|
-
done 'Sidekiq shut down forcefully.'
|
93
|
-
end
|
94
|
-
alias_method :shutdown, :stop
|
95
|
-
|
96
|
-
class Status
|
97
|
-
VALID_SECTIONS = %w[all version overview processes queues]
|
98
|
-
def display(section = nil)
|
99
|
-
section ||= 'all'
|
100
|
-
unless VALID_SECTIONS.include? section
|
101
|
-
puts "I don't know how to check the status of '#{section}'!"
|
102
|
-
puts "Try one of these: #{VALID_SECTIONS.join(', ')}"
|
103
|
-
return
|
104
|
-
end
|
105
|
-
send(section)
|
106
|
-
rescue StandardError => e
|
107
|
-
puts "Couldn't get status: #{e}"
|
108
|
-
end
|
109
|
-
|
110
|
-
def all
|
111
|
-
version
|
112
|
-
puts
|
113
|
-
overview
|
114
|
-
puts
|
115
|
-
processes
|
116
|
-
puts
|
117
|
-
queues
|
118
|
-
end
|
119
|
-
|
120
|
-
def version
|
121
|
-
puts "Sidekiq #{Sidekiq::VERSION}"
|
122
|
-
puts Time.now
|
123
|
-
end
|
124
|
-
|
125
|
-
def overview
|
126
|
-
puts '---- Overview ----'
|
127
|
-
puts " Processed: #{delimit stats.processed}"
|
128
|
-
puts " Failed: #{delimit stats.failed}"
|
129
|
-
puts " Busy: #{delimit stats.workers_size}"
|
130
|
-
puts " Enqueued: #{delimit stats.enqueued}"
|
131
|
-
puts " Retries: #{delimit stats.retry_size}"
|
132
|
-
puts " Scheduled: #{delimit stats.scheduled_size}"
|
133
|
-
puts " Dead: #{delimit stats.dead_size}"
|
134
|
-
end
|
135
|
-
|
136
|
-
def processes
|
137
|
-
puts "---- Processes (#{process_set.size}) ----"
|
138
|
-
process_set.each_with_index do |process, index|
|
139
|
-
puts "#{process['identity']} #{tags_for(process)}"
|
140
|
-
puts " Started: #{Time.at(process['started_at'])} (#{time_ago(process['started_at'])})"
|
141
|
-
puts " Threads: #{process['concurrency']} (#{process['busy']} busy)"
|
142
|
-
puts " Queues: #{split_multiline(process['queues'].sort, pad: 11)}"
|
143
|
-
puts '' unless (index+1) == process_set.size
|
144
|
-
end
|
145
|
-
end
|
146
|
-
|
147
|
-
COL_PAD = 2
|
148
|
-
def queues
|
149
|
-
puts "---- Queues (#{queue_data.size}) ----"
|
150
|
-
columns = {
|
151
|
-
name: [:ljust, (['name'] + queue_data.map(&:name)).map(&:length).max + COL_PAD],
|
152
|
-
size: [:rjust, (['size'] + queue_data.map(&:size)).map(&:length).max + COL_PAD],
|
153
|
-
latency: [:rjust, (['latency'] + queue_data.map(&:latency)).map(&:length).max + COL_PAD]
|
154
|
-
}
|
155
|
-
columns.each { |col, (dir, width)| print col.to_s.upcase.public_send(dir, width) }
|
156
|
-
puts
|
157
|
-
queue_data.each do |q|
|
158
|
-
columns.each do |col, (dir, width)|
|
159
|
-
print q.send(col).public_send(dir, width)
|
160
|
-
end
|
161
|
-
puts
|
162
|
-
end
|
163
|
-
end
|
164
|
-
|
165
|
-
private
|
166
|
-
|
167
|
-
def delimit(number)
|
168
|
-
number.to_s.reverse.scan(/.{1,3}/).join(',').reverse
|
169
|
-
end
|
170
|
-
|
171
|
-
def split_multiline(values, opts = {})
|
172
|
-
return 'none' unless values
|
173
|
-
pad = opts[:pad] || 0
|
174
|
-
max_length = opts[:max_length] || (80 - pad)
|
175
|
-
out = []
|
176
|
-
line = ''
|
177
|
-
values.each do |value|
|
178
|
-
if (line.length + value.length) > max_length
|
179
|
-
out << line
|
180
|
-
line = ' ' * pad
|
181
|
-
end
|
182
|
-
line << value + ', '
|
183
|
-
end
|
184
|
-
out << line[0..-3]
|
185
|
-
out.join("\n")
|
186
|
-
end
|
187
|
-
|
188
|
-
def tags_for(process)
|
189
|
-
tags = [
|
190
|
-
process['tag'],
|
191
|
-
process['labels'],
|
192
|
-
(process['quiet'] == 'true' ? 'quiet' : nil)
|
193
|
-
].flatten.compact
|
194
|
-
tags.any? ? "[#{tags.join('] [')}]" : nil
|
195
|
-
end
|
196
|
-
|
197
|
-
def time_ago(timestamp)
|
198
|
-
seconds = Time.now - Time.at(timestamp)
|
199
|
-
return 'just now' if seconds < 60
|
200
|
-
return 'a minute ago' if seconds < 120
|
201
|
-
return "#{seconds.floor / 60} minutes ago" if seconds < 3600
|
202
|
-
return 'an hour ago' if seconds < 7200
|
203
|
-
"#{seconds.floor / 60 / 60} hours ago"
|
204
|
-
end
|
205
|
-
|
206
|
-
QUEUE_STRUCT = Struct.new(:name, :size, :latency)
|
207
|
-
def queue_data
|
208
|
-
@queue_data ||= Sidekiq::Queue.all.map do |q|
|
209
|
-
QUEUE_STRUCT.new(q.name, q.size.to_s, sprintf('%#.2f', q.latency))
|
210
|
-
end
|
211
|
-
end
|
212
|
-
|
213
|
-
def process_set
|
214
|
-
@process_set ||= Sidekiq::ProcessSet.new
|
215
|
-
end
|
216
|
-
|
217
|
-
def stats
|
218
|
-
@stats ||= Sidekiq::Stats.new
|
219
|
-
end
|
220
|
-
end
|
221
|
-
end
|
data/lib/sidekiq/logging.rb
DELETED
@@ -1,122 +0,0 @@
|
|
1
|
-
# frozen_string_literal: true
|
2
|
-
require 'time'
|
3
|
-
require 'logger'
|
4
|
-
require 'fcntl'
|
5
|
-
|
6
|
-
module Sidekiq
|
7
|
-
module Logging
|
8
|
-
|
9
|
-
class Pretty < Logger::Formatter
|
10
|
-
SPACE = " "
|
11
|
-
|
12
|
-
# Provide a call() method that returns the formatted message.
|
13
|
-
def call(severity, time, program_name, message)
|
14
|
-
"#{time.utc.iso8601(3)} #{::Process.pid} TID-#{Sidekiq::Logging.tid}#{context} #{severity}: #{message}\n"
|
15
|
-
end
|
16
|
-
|
17
|
-
def context
|
18
|
-
c = Thread.current[:sidekiq_context]
|
19
|
-
" #{c.join(SPACE)}" if c && c.any?
|
20
|
-
end
|
21
|
-
end
|
22
|
-
|
23
|
-
class WithoutTimestamp < Pretty
|
24
|
-
def call(severity, time, program_name, message)
|
25
|
-
"#{::Process.pid} TID-#{Sidekiq::Logging.tid}#{context} #{severity}: #{message}\n"
|
26
|
-
end
|
27
|
-
end
|
28
|
-
|
29
|
-
def self.tid
|
30
|
-
Thread.current['sidekiq_tid'] ||= (Thread.current.object_id ^ ::Process.pid).to_s(36)
|
31
|
-
end
|
32
|
-
|
33
|
-
def self.job_hash_context(job_hash)
|
34
|
-
# If we're using a wrapper class, like ActiveJob, use the "wrapped"
|
35
|
-
# attribute to expose the underlying thing.
|
36
|
-
klass = job_hash['wrapped'] || job_hash["class"]
|
37
|
-
bid = job_hash['bid']
|
38
|
-
"#{klass} JID-#{job_hash['jid']}#{" BID-#{bid}" if bid}"
|
39
|
-
end
|
40
|
-
|
41
|
-
def self.with_job_hash_context(job_hash, &block)
|
42
|
-
with_context(job_hash_context(job_hash), &block)
|
43
|
-
end
|
44
|
-
|
45
|
-
def self.with_context(msg)
|
46
|
-
Thread.current[:sidekiq_context] ||= []
|
47
|
-
Thread.current[:sidekiq_context] << msg
|
48
|
-
yield
|
49
|
-
ensure
|
50
|
-
Thread.current[:sidekiq_context].pop
|
51
|
-
end
|
52
|
-
|
53
|
-
def self.initialize_logger(log_target = STDOUT)
|
54
|
-
oldlogger = defined?(@logger) ? @logger : nil
|
55
|
-
@logger = Logger.new(log_target)
|
56
|
-
@logger.level = Logger::INFO
|
57
|
-
@logger.formatter = ENV['DYNO'] ? WithoutTimestamp.new : Pretty.new
|
58
|
-
oldlogger.close if oldlogger && !$TESTING # don't want to close testing's STDOUT logging
|
59
|
-
@logger
|
60
|
-
end
|
61
|
-
|
62
|
-
def self.logger
|
63
|
-
defined?(@logger) ? @logger : initialize_logger
|
64
|
-
end
|
65
|
-
|
66
|
-
def self.logger=(log)
|
67
|
-
@logger = (log ? log : Logger.new(File::NULL))
|
68
|
-
end
|
69
|
-
|
70
|
-
# This reopens ALL logfiles in the process that have been rotated
|
71
|
-
# using logrotate(8) (without copytruncate) or similar tools.
|
72
|
-
# A +File+ object is considered for reopening if it is:
|
73
|
-
# 1) opened with the O_APPEND and O_WRONLY flags
|
74
|
-
# 2) the current open file handle does not match its original open path
|
75
|
-
# 3) unbuffered (as far as userspace buffering goes, not O_SYNC)
|
76
|
-
# Returns the number of files reopened
|
77
|
-
def self.reopen_logs
|
78
|
-
to_reopen = []
|
79
|
-
append_flags = File::WRONLY | File::APPEND
|
80
|
-
|
81
|
-
ObjectSpace.each_object(File) do |fp|
|
82
|
-
begin
|
83
|
-
if !fp.closed? && fp.stat.file? && fp.sync && (fp.fcntl(Fcntl::F_GETFL) & append_flags) == append_flags
|
84
|
-
to_reopen << fp
|
85
|
-
end
|
86
|
-
rescue IOError, Errno::EBADF
|
87
|
-
end
|
88
|
-
end
|
89
|
-
|
90
|
-
nr = 0
|
91
|
-
to_reopen.each do |fp|
|
92
|
-
orig_st = begin
|
93
|
-
fp.stat
|
94
|
-
rescue IOError, Errno::EBADF
|
95
|
-
next
|
96
|
-
end
|
97
|
-
|
98
|
-
begin
|
99
|
-
b = File.stat(fp.path)
|
100
|
-
next if orig_st.ino == b.ino && orig_st.dev == b.dev
|
101
|
-
rescue Errno::ENOENT
|
102
|
-
end
|
103
|
-
|
104
|
-
begin
|
105
|
-
File.open(fp.path, 'a') { |tmpfp| fp.reopen(tmpfp) }
|
106
|
-
fp.sync = true
|
107
|
-
nr += 1
|
108
|
-
rescue IOError, Errno::EBADF
|
109
|
-
# not much we can do...
|
110
|
-
end
|
111
|
-
end
|
112
|
-
nr
|
113
|
-
rescue RuntimeError => ex
|
114
|
-
# RuntimeError: ObjectSpace is disabled; each_object will only work with Class, pass -X+O to enable
|
115
|
-
puts "Unable to reopen logs: #{ex.message}"
|
116
|
-
end
|
117
|
-
|
118
|
-
def logger
|
119
|
-
Sidekiq::Logging.logger
|
120
|
-
end
|
121
|
-
end
|
122
|
-
end
|
@@ -1,23 +0,0 @@
|
|
1
|
-
# frozen_string_literal: true
|
2
|
-
module Sidekiq
|
3
|
-
module Middleware
|
4
|
-
module Server
|
5
|
-
class ActiveRecord
|
6
|
-
|
7
|
-
def initialize
|
8
|
-
# With Rails 5+ we must use the Reloader **always**.
|
9
|
-
# The reloader handles code loading and db connection management.
|
10
|
-
if defined?(::Rails) && ::Rails::VERSION::MAJOR >= 5
|
11
|
-
raise ArgumentError, "Rails 5 no longer needs or uses the ActiveRecord middleware."
|
12
|
-
end
|
13
|
-
end
|
14
|
-
|
15
|
-
def call(*args)
|
16
|
-
yield
|
17
|
-
ensure
|
18
|
-
::ActiveRecord::Base.clear_active_connections!
|
19
|
-
end
|
20
|
-
end
|
21
|
-
end
|
22
|
-
end
|
23
|
-
end
|