sidekiq 5.2.10 → 6.5.6
Sign up to get free protection for your applications and to get access to all the features.
Potentially problematic release.
This version of sidekiq might be problematic. Click here for more details.
- checksums.yaml +4 -4
- data/Changes.md +391 -1
- data/LICENSE +3 -3
- data/README.md +24 -35
- data/bin/sidekiq +27 -3
- data/bin/sidekiqload +79 -67
- data/bin/sidekiqmon +8 -0
- data/lib/generators/sidekiq/job_generator.rb +57 -0
- data/lib/generators/sidekiq/templates/{worker.rb.erb → job.rb.erb} +2 -2
- data/lib/generators/sidekiq/templates/{worker_spec.rb.erb → job_spec.rb.erb} +1 -1
- data/lib/generators/sidekiq/templates/{worker_test.rb.erb → job_test.rb.erb} +1 -1
- data/lib/sidekiq/api.rb +504 -307
- data/lib/sidekiq/cli.rb +190 -206
- data/lib/sidekiq/client.rb +77 -81
- data/lib/sidekiq/component.rb +65 -0
- data/lib/sidekiq/delay.rb +8 -7
- data/lib/sidekiq/extensions/action_mailer.rb +13 -22
- data/lib/sidekiq/extensions/active_record.rb +13 -10
- data/lib/sidekiq/extensions/class_methods.rb +14 -11
- data/lib/sidekiq/extensions/generic_proxy.rb +7 -5
- data/lib/sidekiq/fetch.rb +50 -40
- data/lib/sidekiq/job.rb +13 -0
- data/lib/sidekiq/job_logger.rb +33 -7
- data/lib/sidekiq/job_retry.rb +126 -106
- data/lib/sidekiq/job_util.rb +71 -0
- data/lib/sidekiq/launcher.rb +177 -83
- data/lib/sidekiq/logger.rb +156 -0
- data/lib/sidekiq/manager.rb +40 -41
- data/lib/sidekiq/metrics/deploy.rb +47 -0
- data/lib/sidekiq/metrics/query.rb +153 -0
- data/lib/sidekiq/metrics/shared.rb +94 -0
- data/lib/sidekiq/metrics/tracking.rb +134 -0
- data/lib/sidekiq/middleware/chain.rb +102 -46
- data/lib/sidekiq/middleware/current_attributes.rb +63 -0
- data/lib/sidekiq/middleware/i18n.rb +7 -7
- data/lib/sidekiq/middleware/modules.rb +21 -0
- data/lib/sidekiq/monitor.rb +133 -0
- data/lib/sidekiq/paginator.rb +20 -16
- data/lib/sidekiq/processor.rb +104 -97
- data/lib/sidekiq/rails.rb +47 -37
- data/lib/sidekiq/redis_client_adapter.rb +154 -0
- data/lib/sidekiq/redis_connection.rb +108 -77
- data/lib/sidekiq/ring_buffer.rb +29 -0
- data/lib/sidekiq/scheduled.rb +64 -35
- data/lib/sidekiq/sd_notify.rb +149 -0
- data/lib/sidekiq/systemd.rb +24 -0
- data/lib/sidekiq/testing/inline.rb +6 -5
- data/lib/sidekiq/testing.rb +68 -58
- data/lib/sidekiq/transaction_aware_client.rb +45 -0
- data/lib/sidekiq/version.rb +2 -1
- data/lib/sidekiq/web/action.rb +15 -11
- data/lib/sidekiq/web/application.rb +100 -77
- data/lib/sidekiq/web/csrf_protection.rb +180 -0
- data/lib/sidekiq/web/helpers.rb +134 -94
- data/lib/sidekiq/web/router.rb +23 -19
- data/lib/sidekiq/web.rb +65 -105
- data/lib/sidekiq/worker.rb +253 -106
- data/lib/sidekiq.rb +170 -62
- data/sidekiq.gemspec +23 -16
- data/web/assets/images/apple-touch-icon.png +0 -0
- data/web/assets/javascripts/application.js +112 -61
- data/web/assets/javascripts/chart.min.js +13 -0
- data/web/assets/javascripts/chartjs-plugin-annotation.min.js +7 -0
- data/web/assets/javascripts/dashboard.js +53 -89
- data/web/assets/javascripts/graph.js +16 -0
- data/web/assets/javascripts/metrics.js +262 -0
- data/web/assets/stylesheets/application-dark.css +143 -0
- data/web/assets/stylesheets/application-rtl.css +0 -4
- data/web/assets/stylesheets/application.css +88 -233
- data/web/locales/ar.yml +8 -2
- data/web/locales/de.yml +14 -2
- data/web/locales/el.yml +43 -19
- data/web/locales/en.yml +13 -1
- data/web/locales/es.yml +18 -2
- data/web/locales/fr.yml +10 -3
- data/web/locales/ja.yml +7 -1
- data/web/locales/lt.yml +83 -0
- data/web/locales/pl.yml +4 -4
- data/web/locales/pt-br.yml +27 -9
- data/web/locales/ru.yml +4 -0
- data/web/locales/vi.yml +83 -0
- data/web/views/_footer.erb +1 -1
- data/web/views/_job_info.erb +3 -2
- data/web/views/_nav.erb +1 -1
- data/web/views/_poll_link.erb +2 -5
- data/web/views/_summary.erb +7 -7
- data/web/views/busy.erb +56 -22
- data/web/views/dashboard.erb +23 -14
- data/web/views/dead.erb +3 -3
- data/web/views/layout.erb +3 -1
- data/web/views/metrics.erb +69 -0
- data/web/views/metrics_for_job.erb +87 -0
- data/web/views/morgue.erb +9 -6
- data/web/views/queue.erb +23 -10
- data/web/views/queues.erb +10 -2
- data/web/views/retries.erb +11 -8
- data/web/views/retry.erb +3 -3
- data/web/views/scheduled.erb +5 -2
- metadata +53 -64
- data/.circleci/config.yml +0 -61
- data/.github/contributing.md +0 -32
- data/.github/issue_template.md +0 -11
- data/.gitignore +0 -15
- data/.travis.yml +0 -11
- data/3.0-Upgrade.md +0 -70
- data/4.0-Upgrade.md +0 -53
- data/5.0-Upgrade.md +0 -56
- data/COMM-LICENSE +0 -97
- data/Ent-Changes.md +0 -238
- data/Gemfile +0 -19
- data/Pro-2.0-Upgrade.md +0 -138
- data/Pro-3.0-Upgrade.md +0 -44
- data/Pro-4.0-Upgrade.md +0 -35
- data/Pro-Changes.md +0 -759
- data/Rakefile +0 -9
- data/bin/sidekiqctl +0 -20
- data/code_of_conduct.md +0 -50
- data/lib/generators/sidekiq/worker_generator.rb +0 -49
- data/lib/sidekiq/core_ext.rb +0 -1
- data/lib/sidekiq/ctl.rb +0 -221
- data/lib/sidekiq/exception_handler.rb +0 -29
- data/lib/sidekiq/logging.rb +0 -122
- data/lib/sidekiq/middleware/server/active_record.rb +0 -23
- data/lib/sidekiq/util.rb +0 -66
data/lib/sidekiq/launcher.rb
CHANGED
@@ -1,25 +1,33 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
|
-
|
3
|
-
require
|
4
|
-
require
|
2
|
+
|
3
|
+
require "sidekiq/manager"
|
4
|
+
require "sidekiq/fetch"
|
5
|
+
require "sidekiq/scheduled"
|
6
|
+
require "sidekiq/ring_buffer"
|
5
7
|
|
6
8
|
module Sidekiq
|
7
|
-
# The Launcher
|
8
|
-
# start, monitor and stop the core Actors in Sidekiq.
|
9
|
-
# If any of these actors die, the Sidekiq process exits
|
10
|
-
# immediately.
|
9
|
+
# The Launcher starts the Manager and Poller threads and provides the process heartbeat.
|
11
10
|
class Launcher
|
12
|
-
include
|
11
|
+
include Sidekiq::Component
|
13
12
|
|
14
|
-
|
13
|
+
STATS_TTL = 5 * 365 * 24 * 60 * 60 # 5 years
|
15
14
|
|
16
|
-
|
15
|
+
PROCTITLES = [
|
16
|
+
proc { "sidekiq" },
|
17
|
+
proc { Sidekiq::VERSION },
|
18
|
+
proc { |me, data| data["tag"] },
|
19
|
+
proc { |me, data| "[#{Processor::WORK_STATE.size} of #{data["concurrency"]} busy]" },
|
20
|
+
proc { |me, data| "stopping" if me.stopping? }
|
21
|
+
]
|
22
|
+
|
23
|
+
attr_accessor :manager, :poller, :fetcher
|
17
24
|
|
18
25
|
def initialize(options)
|
26
|
+
@config = options
|
27
|
+
options[:fetch] ||= BasicFetch.new(options)
|
19
28
|
@manager = Sidekiq::Manager.new(options)
|
20
|
-
@poller = Sidekiq::Scheduled::Poller.new
|
29
|
+
@poller = Sidekiq::Scheduled::Poller.new(options)
|
21
30
|
@done = false
|
22
|
-
@options = options
|
23
31
|
end
|
24
32
|
|
25
33
|
def run
|
@@ -36,11 +44,9 @@ module Sidekiq
|
|
36
44
|
@poller.terminate
|
37
45
|
end
|
38
46
|
|
39
|
-
# Shuts down
|
40
|
-
# return until all work is complete and cleaned up.
|
41
|
-
# It can take up to the timeout to complete.
|
47
|
+
# Shuts down this Sidekiq instance. Waits up to the deadline for all jobs to complete.
|
42
48
|
def stop
|
43
|
-
deadline = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC) + @
|
49
|
+
deadline = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC) + @config[:timeout]
|
44
50
|
|
45
51
|
@done = true
|
46
52
|
@manager.quiet
|
@@ -48,10 +54,10 @@ module Sidekiq
|
|
48
54
|
|
49
55
|
@manager.stop(deadline)
|
50
56
|
|
51
|
-
# Requeue everything in case there was a
|
57
|
+
# Requeue everything in case there was a thread which fetched a job while the process was stopped.
|
52
58
|
# This call is a no-op in Sidekiq but necessary for Sidekiq Pro.
|
53
|
-
strategy =
|
54
|
-
strategy.bulk_requeue([], @
|
59
|
+
strategy = @config[:fetch]
|
60
|
+
strategy.bulk_requeue([], @config)
|
55
61
|
|
56
62
|
clear_heartbeat
|
57
63
|
end
|
@@ -62,112 +68,200 @@ module Sidekiq
|
|
62
68
|
|
63
69
|
private unless $TESTING
|
64
70
|
|
71
|
+
BEAT_PAUSE = 5
|
72
|
+
|
73
|
+
def start_heartbeat
|
74
|
+
loop do
|
75
|
+
heartbeat
|
76
|
+
sleep BEAT_PAUSE
|
77
|
+
end
|
78
|
+
logger.info("Heartbeat stopping...")
|
79
|
+
end
|
80
|
+
|
81
|
+
def clear_heartbeat
|
82
|
+
flush_stats
|
83
|
+
|
84
|
+
# Remove record from Redis since we are shutting down.
|
85
|
+
# Note we don't stop the heartbeat thread; if the process
|
86
|
+
# doesn't actually exit, it'll reappear in the Web UI.
|
87
|
+
redis do |conn|
|
88
|
+
conn.pipelined do |pipeline|
|
89
|
+
pipeline.srem("processes", [identity])
|
90
|
+
pipeline.unlink("#{identity}:work")
|
91
|
+
end
|
92
|
+
end
|
93
|
+
rescue
|
94
|
+
# best effort, ignore network errors
|
95
|
+
end
|
96
|
+
|
65
97
|
def heartbeat
|
66
|
-
|
67
|
-
results.compact!
|
68
|
-
$0 = results.join(' ')
|
98
|
+
$0 = PROCTITLES.map { |proc| proc.call(self, to_data) }.compact.join(" ")
|
69
99
|
|
70
100
|
❤
|
71
101
|
end
|
72
102
|
|
103
|
+
def flush_stats
|
104
|
+
fails = Processor::FAILURE.reset
|
105
|
+
procd = Processor::PROCESSED.reset
|
106
|
+
return if fails + procd == 0
|
107
|
+
|
108
|
+
nowdate = Time.now.utc.strftime("%Y-%m-%d")
|
109
|
+
begin
|
110
|
+
Sidekiq.redis do |conn|
|
111
|
+
conn.pipelined do |pipeline|
|
112
|
+
pipeline.incrby("stat:processed", procd)
|
113
|
+
pipeline.incrby("stat:processed:#{nowdate}", procd)
|
114
|
+
pipeline.expire("stat:processed:#{nowdate}", STATS_TTL)
|
115
|
+
|
116
|
+
pipeline.incrby("stat:failed", fails)
|
117
|
+
pipeline.incrby("stat:failed:#{nowdate}", fails)
|
118
|
+
pipeline.expire("stat:failed:#{nowdate}", STATS_TTL)
|
119
|
+
end
|
120
|
+
end
|
121
|
+
rescue => ex
|
122
|
+
# we're exiting the process, things might be shut down so don't
|
123
|
+
# try to handle the exception
|
124
|
+
Sidekiq.logger.warn("Unable to flush stats: #{ex}")
|
125
|
+
end
|
126
|
+
end
|
127
|
+
|
73
128
|
def ❤
|
74
129
|
key = identity
|
75
130
|
fails = procd = 0
|
131
|
+
|
76
132
|
begin
|
77
133
|
fails = Processor::FAILURE.reset
|
78
134
|
procd = Processor::PROCESSED.reset
|
79
|
-
curstate = Processor::
|
135
|
+
curstate = Processor::WORK_STATE.dup
|
80
136
|
|
81
|
-
workers_key = "#{key}:workers"
|
82
137
|
nowdate = Time.now.utc.strftime("%Y-%m-%d")
|
83
|
-
Sidekiq.redis do |conn|
|
84
|
-
conn.multi do
|
85
|
-
conn.incrby("stat:processed", procd)
|
86
|
-
conn.incrby("stat:processed:#{nowdate}", procd)
|
87
|
-
conn.expire("stat:processed:#{nowdate}", STATS_TTL)
|
88
138
|
|
89
|
-
|
90
|
-
|
91
|
-
|
139
|
+
redis do |conn|
|
140
|
+
conn.multi do |transaction|
|
141
|
+
transaction.incrby("stat:processed", procd)
|
142
|
+
transaction.incrby("stat:processed:#{nowdate}", procd)
|
143
|
+
transaction.expire("stat:processed:#{nowdate}", STATS_TTL)
|
144
|
+
|
145
|
+
transaction.incrby("stat:failed", fails)
|
146
|
+
transaction.incrby("stat:failed:#{nowdate}", fails)
|
147
|
+
transaction.expire("stat:failed:#{nowdate}", STATS_TTL)
|
148
|
+
end
|
92
149
|
|
93
|
-
|
150
|
+
# work is the current set of executing jobs
|
151
|
+
work_key = "#{key}:work"
|
152
|
+
conn.pipelined do |transaction|
|
153
|
+
transaction.unlink(work_key)
|
94
154
|
curstate.each_pair do |tid, hash|
|
95
|
-
|
155
|
+
transaction.hset(work_key, tid, Sidekiq.dump_json(hash))
|
96
156
|
end
|
97
|
-
|
157
|
+
transaction.expire(work_key, 60)
|
98
158
|
end
|
99
159
|
end
|
160
|
+
|
161
|
+
rtt = check_rtt
|
162
|
+
|
100
163
|
fails = procd = 0
|
164
|
+
kb = memory_usage(::Process.pid)
|
101
165
|
|
102
|
-
_, exists, _, _, msg =
|
103
|
-
conn.multi
|
104
|
-
|
105
|
-
|
106
|
-
|
107
|
-
|
108
|
-
|
109
|
-
|
110
|
-
|
166
|
+
_, exists, _, _, msg = redis { |conn|
|
167
|
+
conn.multi { |transaction|
|
168
|
+
transaction.sadd("processes", [key])
|
169
|
+
transaction.exists?(key)
|
170
|
+
transaction.hmset(key, "info", to_json,
|
171
|
+
"busy", curstate.size,
|
172
|
+
"beat", Time.now.to_f,
|
173
|
+
"rtt_us", rtt,
|
174
|
+
"quiet", @done.to_s,
|
175
|
+
"rss", kb)
|
176
|
+
transaction.expire(key, 60)
|
177
|
+
transaction.rpop("#{key}-signals")
|
178
|
+
}
|
179
|
+
}
|
111
180
|
|
112
181
|
# first heartbeat or recovering from an outage and need to reestablish our heartbeat
|
113
|
-
fire_event(:heartbeat)
|
182
|
+
fire_event(:heartbeat) unless exists
|
183
|
+
fire_event(:beat, oneshot: false)
|
114
184
|
|
115
185
|
return unless msg
|
116
186
|
|
117
|
-
::Process.kill(msg,
|
187
|
+
::Process.kill(msg, ::Process.pid)
|
118
188
|
rescue => e
|
119
189
|
# ignore all redis/network issues
|
120
|
-
logger.error("heartbeat: #{e
|
190
|
+
logger.error("heartbeat: #{e}")
|
121
191
|
# don't lose the counts if there was a network issue
|
122
192
|
Processor::PROCESSED.incr(procd)
|
123
193
|
Processor::FAILURE.incr(fails)
|
124
194
|
end
|
125
195
|
end
|
126
196
|
|
127
|
-
|
128
|
-
|
129
|
-
|
130
|
-
|
197
|
+
# We run the heartbeat every five seconds.
|
198
|
+
# Capture five samples of RTT, log a warning if each sample
|
199
|
+
# is above our warning threshold.
|
200
|
+
RTT_READINGS = RingBuffer.new(5)
|
201
|
+
RTT_WARNING_LEVEL = 50_000
|
202
|
+
|
203
|
+
def check_rtt
|
204
|
+
a = b = 0
|
205
|
+
redis do |x|
|
206
|
+
a = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC, :microsecond)
|
207
|
+
x.ping
|
208
|
+
b = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC, :microsecond)
|
131
209
|
end
|
132
|
-
|
210
|
+
rtt = b - a
|
211
|
+
RTT_READINGS << rtt
|
212
|
+
# Ideal RTT for Redis is < 1000µs
|
213
|
+
# Workable is < 10,000µs
|
214
|
+
# Log a warning if it's a disaster.
|
215
|
+
if RTT_READINGS.all? { |x| x > RTT_WARNING_LEVEL }
|
216
|
+
logger.warn <<~EOM
|
217
|
+
Your Redis network connection is performing extremely poorly.
|
218
|
+
Last RTT readings were #{RTT_READINGS.buffer.inspect}, ideally these should be < 1000.
|
219
|
+
Ensure Redis is running in the same AZ or datacenter as Sidekiq.
|
220
|
+
If these values are close to 100,000, that means your Sidekiq process may be
|
221
|
+
CPU-saturated; reduce your concurrency and/or see https://github.com/mperham/sidekiq/discussions/5039
|
222
|
+
EOM
|
223
|
+
RTT_READINGS.reset
|
224
|
+
end
|
225
|
+
rtt
|
133
226
|
end
|
134
227
|
|
135
|
-
|
136
|
-
|
137
|
-
|
138
|
-
|
139
|
-
|
140
|
-
|
141
|
-
|
142
|
-
|
143
|
-
|
144
|
-
|
145
|
-
|
146
|
-
|
147
|
-
|
228
|
+
MEMORY_GRABBER = case RUBY_PLATFORM
|
229
|
+
when /linux/
|
230
|
+
->(pid) {
|
231
|
+
IO.readlines("/proc/#{$$}/status").each do |line|
|
232
|
+
next unless line.start_with?("VmRSS:")
|
233
|
+
break line.split[1].to_i
|
234
|
+
end
|
235
|
+
}
|
236
|
+
when /darwin|bsd/
|
237
|
+
->(pid) {
|
238
|
+
`ps -o pid,rss -p #{pid}`.lines.last.split.last.to_i
|
239
|
+
}
|
240
|
+
else
|
241
|
+
->(pid) { 0 }
|
148
242
|
end
|
149
243
|
|
150
|
-
def
|
151
|
-
|
152
|
-
# this data changes infrequently so dump it to a string
|
153
|
-
# now so we don't need to dump it every heartbeat.
|
154
|
-
Sidekiq.dump_json(to_data)
|
155
|
-
end
|
244
|
+
def memory_usage(pid)
|
245
|
+
MEMORY_GRABBER.call(pid)
|
156
246
|
end
|
157
247
|
|
158
|
-
def
|
159
|
-
|
160
|
-
|
161
|
-
|
162
|
-
|
163
|
-
|
164
|
-
|
165
|
-
|
166
|
-
|
167
|
-
|
168
|
-
|
169
|
-
# best effort, ignore network errors
|
248
|
+
def to_data
|
249
|
+
@data ||= {
|
250
|
+
"hostname" => hostname,
|
251
|
+
"started_at" => Time.now.to_f,
|
252
|
+
"pid" => ::Process.pid,
|
253
|
+
"tag" => @config[:tag] || "",
|
254
|
+
"concurrency" => @config[:concurrency],
|
255
|
+
"queues" => @config[:queues].uniq,
|
256
|
+
"labels" => @config[:labels],
|
257
|
+
"identity" => identity
|
258
|
+
}
|
170
259
|
end
|
171
260
|
|
261
|
+
def to_json
|
262
|
+
# this data changes infrequently so dump it to a string
|
263
|
+
# now so we don't need to dump it every heartbeat.
|
264
|
+
@json ||= Sidekiq.dump_json(to_data)
|
265
|
+
end
|
172
266
|
end
|
173
267
|
end
|
@@ -0,0 +1,156 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require "logger"
|
4
|
+
require "time"
|
5
|
+
|
6
|
+
module Sidekiq
|
7
|
+
module Context
|
8
|
+
def self.with(hash)
|
9
|
+
orig_context = current.dup
|
10
|
+
current.merge!(hash)
|
11
|
+
yield
|
12
|
+
ensure
|
13
|
+
Thread.current[:sidekiq_context] = orig_context
|
14
|
+
end
|
15
|
+
|
16
|
+
def self.current
|
17
|
+
Thread.current[:sidekiq_context] ||= {}
|
18
|
+
end
|
19
|
+
|
20
|
+
def self.add(k, v)
|
21
|
+
current[k] = v
|
22
|
+
end
|
23
|
+
end
|
24
|
+
|
25
|
+
module LoggingUtils
|
26
|
+
LEVELS = {
|
27
|
+
"debug" => 0,
|
28
|
+
"info" => 1,
|
29
|
+
"warn" => 2,
|
30
|
+
"error" => 3,
|
31
|
+
"fatal" => 4
|
32
|
+
}
|
33
|
+
LEVELS.default_proc = proc do |_, level|
|
34
|
+
Sidekiq.logger.warn("Invalid log level: #{level.inspect}")
|
35
|
+
nil
|
36
|
+
end
|
37
|
+
|
38
|
+
LEVELS.each do |level, numeric_level|
|
39
|
+
define_method("#{level}?") do
|
40
|
+
local_level.nil? ? super() : local_level <= numeric_level
|
41
|
+
end
|
42
|
+
end
|
43
|
+
|
44
|
+
def local_level
|
45
|
+
Thread.current[:sidekiq_log_level]
|
46
|
+
end
|
47
|
+
|
48
|
+
def local_level=(level)
|
49
|
+
case level
|
50
|
+
when Integer
|
51
|
+
Thread.current[:sidekiq_log_level] = level
|
52
|
+
when Symbol, String
|
53
|
+
Thread.current[:sidekiq_log_level] = LEVELS[level.to_s]
|
54
|
+
when nil
|
55
|
+
Thread.current[:sidekiq_log_level] = nil
|
56
|
+
else
|
57
|
+
raise ArgumentError, "Invalid log level: #{level.inspect}"
|
58
|
+
end
|
59
|
+
end
|
60
|
+
|
61
|
+
def level
|
62
|
+
local_level || super
|
63
|
+
end
|
64
|
+
|
65
|
+
# Change the thread-local level for the duration of the given block.
|
66
|
+
def log_at(level)
|
67
|
+
old_local_level = local_level
|
68
|
+
self.local_level = level
|
69
|
+
yield
|
70
|
+
ensure
|
71
|
+
self.local_level = old_local_level
|
72
|
+
end
|
73
|
+
|
74
|
+
# Redefined to check severity against #level, and thus the thread-local level, rather than +@level+.
|
75
|
+
# FIXME: Remove when the minimum Ruby version supports overriding Logger#level.
|
76
|
+
def add(severity, message = nil, progname = nil, &block)
|
77
|
+
severity ||= ::Logger::UNKNOWN
|
78
|
+
progname ||= @progname
|
79
|
+
|
80
|
+
return true if @logdev.nil? || severity < level
|
81
|
+
|
82
|
+
if message.nil?
|
83
|
+
if block
|
84
|
+
message = yield
|
85
|
+
else
|
86
|
+
message = progname
|
87
|
+
progname = @progname
|
88
|
+
end
|
89
|
+
end
|
90
|
+
|
91
|
+
@logdev.write format_message(format_severity(severity), Time.now, progname, message)
|
92
|
+
end
|
93
|
+
end
|
94
|
+
|
95
|
+
class Logger < ::Logger
|
96
|
+
include LoggingUtils
|
97
|
+
|
98
|
+
def initialize(*args, **kwargs)
|
99
|
+
super
|
100
|
+
self.formatter = Sidekiq.log_formatter
|
101
|
+
end
|
102
|
+
|
103
|
+
module Formatters
|
104
|
+
class Base < ::Logger::Formatter
|
105
|
+
def tid
|
106
|
+
Thread.current["sidekiq_tid"] ||= (Thread.current.object_id ^ ::Process.pid).to_s(36)
|
107
|
+
end
|
108
|
+
|
109
|
+
def ctx
|
110
|
+
Sidekiq::Context.current
|
111
|
+
end
|
112
|
+
|
113
|
+
def format_context
|
114
|
+
if ctx.any?
|
115
|
+
" " + ctx.compact.map { |k, v|
|
116
|
+
case v
|
117
|
+
when Array
|
118
|
+
"#{k}=#{v.join(",")}"
|
119
|
+
else
|
120
|
+
"#{k}=#{v}"
|
121
|
+
end
|
122
|
+
}.join(" ")
|
123
|
+
end
|
124
|
+
end
|
125
|
+
end
|
126
|
+
|
127
|
+
class Pretty < Base
|
128
|
+
def call(severity, time, program_name, message)
|
129
|
+
"#{time.utc.iso8601(3)} pid=#{::Process.pid} tid=#{tid}#{format_context} #{severity}: #{message}\n"
|
130
|
+
end
|
131
|
+
end
|
132
|
+
|
133
|
+
class WithoutTimestamp < Pretty
|
134
|
+
def call(severity, time, program_name, message)
|
135
|
+
"pid=#{::Process.pid} tid=#{tid}#{format_context} #{severity}: #{message}\n"
|
136
|
+
end
|
137
|
+
end
|
138
|
+
|
139
|
+
class JSON < Base
|
140
|
+
def call(severity, time, program_name, message)
|
141
|
+
hash = {
|
142
|
+
ts: time.utc.iso8601(3),
|
143
|
+
pid: ::Process.pid,
|
144
|
+
tid: tid,
|
145
|
+
lvl: severity,
|
146
|
+
msg: message
|
147
|
+
}
|
148
|
+
c = ctx
|
149
|
+
hash["ctx"] = c unless c.empty?
|
150
|
+
|
151
|
+
Sidekiq.dump_json(hash) << "\n"
|
152
|
+
end
|
153
|
+
end
|
154
|
+
end
|
155
|
+
end
|
156
|
+
end
|
data/lib/sidekiq/manager.rb
CHANGED
@@ -1,12 +1,10 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
|
-
require 'sidekiq/util'
|
3
|
-
require 'sidekiq/processor'
|
4
|
-
require 'sidekiq/fetch'
|
5
|
-
require 'thread'
|
6
|
-
require 'set'
|
7
2
|
|
8
|
-
|
3
|
+
require "sidekiq/processor"
|
4
|
+
require "sidekiq/fetch"
|
5
|
+
require "set"
|
9
6
|
|
7
|
+
module Sidekiq
|
10
8
|
##
|
11
9
|
# The Manager is the central coordination point in Sidekiq, controlling
|
12
10
|
# the lifecycle of the Processors.
|
@@ -22,43 +20,37 @@ module Sidekiq
|
|
22
20
|
# the shutdown process. The other tasks are performed by other threads.
|
23
21
|
#
|
24
22
|
class Manager
|
25
|
-
include
|
23
|
+
include Sidekiq::Component
|
26
24
|
|
27
25
|
attr_reader :workers
|
28
|
-
attr_reader :options
|
29
26
|
|
30
|
-
def initialize(options={})
|
27
|
+
def initialize(options = {})
|
28
|
+
@config = options
|
31
29
|
logger.debug { options.inspect }
|
32
|
-
@options = options
|
33
30
|
@count = options[:concurrency] || 10
|
34
31
|
raise ArgumentError, "Concurrency of #{@count} is not supported" if @count < 1
|
35
32
|
|
36
33
|
@done = false
|
37
34
|
@workers = Set.new
|
38
35
|
@count.times do
|
39
|
-
@workers << Processor.new(
|
36
|
+
@workers << Processor.new(@config, &method(:processor_result))
|
40
37
|
end
|
41
38
|
@plock = Mutex.new
|
42
39
|
end
|
43
40
|
|
44
41
|
def start
|
45
|
-
@workers.each
|
46
|
-
x.start
|
47
|
-
end
|
42
|
+
@workers.each(&:start)
|
48
43
|
end
|
49
44
|
|
50
45
|
def quiet
|
51
46
|
return if @done
|
52
47
|
@done = true
|
53
48
|
|
54
|
-
logger.info { "Terminating quiet
|
55
|
-
@workers.each
|
49
|
+
logger.info { "Terminating quiet threads" }
|
50
|
+
@workers.each(&:terminate)
|
56
51
|
fire_event(:quiet, reverse: true)
|
57
52
|
end
|
58
53
|
|
59
|
-
# hack for quicker development / testing environment #2774
|
60
|
-
PAUSE_TIME = STDOUT.tty? ? 0.1 : 0.5
|
61
|
-
|
62
54
|
def stop(deadline)
|
63
55
|
quiet
|
64
56
|
fire_event(:shutdown, reverse: true)
|
@@ -69,29 +61,18 @@ module Sidekiq
|
|
69
61
|
sleep PAUSE_TIME
|
70
62
|
return if @workers.empty?
|
71
63
|
|
72
|
-
logger.info { "Pausing to allow
|
73
|
-
|
74
|
-
while remaining > PAUSE_TIME
|
75
|
-
return if @workers.empty?
|
76
|
-
sleep PAUSE_TIME
|
77
|
-
remaining = deadline - ::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
|
78
|
-
end
|
64
|
+
logger.info { "Pausing to allow jobs to finish..." }
|
65
|
+
wait_for(deadline) { @workers.empty? }
|
79
66
|
return if @workers.empty?
|
80
67
|
|
81
68
|
hard_shutdown
|
82
69
|
end
|
83
70
|
|
84
|
-
def
|
85
|
-
@plock.synchronize do
|
86
|
-
@workers.delete(processor)
|
87
|
-
end
|
88
|
-
end
|
89
|
-
|
90
|
-
def processor_died(processor, reason)
|
71
|
+
def processor_result(processor, reason = nil)
|
91
72
|
@plock.synchronize do
|
92
73
|
@workers.delete(processor)
|
93
74
|
unless @done
|
94
|
-
p = Processor.new(
|
75
|
+
p = Processor.new(@config, &method(:processor_result))
|
95
76
|
@workers << p
|
96
77
|
p.start
|
97
78
|
end
|
@@ -105,7 +86,7 @@ module Sidekiq
|
|
105
86
|
private
|
106
87
|
|
107
88
|
def hard_shutdown
|
108
|
-
# We've reached the timeout and we still have busy
|
89
|
+
# We've reached the timeout and we still have busy threads.
|
109
90
|
# They must die but their jobs shall live on.
|
110
91
|
cleanup = nil
|
111
92
|
@plock.synchronize do
|
@@ -113,25 +94,43 @@ module Sidekiq
|
|
113
94
|
end
|
114
95
|
|
115
96
|
if cleanup.size > 0
|
116
|
-
jobs = cleanup.map {|p| p.job }.compact
|
97
|
+
jobs = cleanup.map { |p| p.job }.compact
|
117
98
|
|
118
|
-
logger.warn { "Terminating #{cleanup.size} busy
|
119
|
-
logger.
|
99
|
+
logger.warn { "Terminating #{cleanup.size} busy threads" }
|
100
|
+
logger.debug { "Jobs still in progress #{jobs.inspect}" }
|
120
101
|
|
121
102
|
# Re-enqueue unfinished jobs
|
122
103
|
# NOTE: You may notice that we may push a job back to redis before
|
123
|
-
# the
|
104
|
+
# the thread is terminated. This is ok because Sidekiq's
|
124
105
|
# contract says that jobs are run AT LEAST once. Process termination
|
125
106
|
# is delayed until we're certain the jobs are back in Redis because
|
126
107
|
# it is worse to lose a job than to run it twice.
|
127
|
-
strategy =
|
128
|
-
strategy.bulk_requeue(jobs, @
|
108
|
+
strategy = @config[:fetch]
|
109
|
+
strategy.bulk_requeue(jobs, @config)
|
129
110
|
end
|
130
111
|
|
131
112
|
cleanup.each do |processor|
|
132
113
|
processor.kill
|
133
114
|
end
|
115
|
+
|
116
|
+
# when this method returns, we immediately call `exit` which may not give
|
117
|
+
# the remaining threads time to run `ensure` blocks, etc. We pause here up
|
118
|
+
# to 3 seconds to give threads a minimal amount of time to run `ensure` blocks.
|
119
|
+
deadline = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC) + 3
|
120
|
+
wait_for(deadline) { @workers.empty? }
|
134
121
|
end
|
135
122
|
|
123
|
+
# hack for quicker development / testing environment #2774
|
124
|
+
PAUSE_TIME = $stdout.tty? ? 0.1 : 0.5
|
125
|
+
|
126
|
+
# Wait for the orblock to be true or the deadline passed.
|
127
|
+
def wait_for(deadline, &condblock)
|
128
|
+
remaining = deadline - ::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
|
129
|
+
while remaining > PAUSE_TIME
|
130
|
+
return if condblock.call
|
131
|
+
sleep PAUSE_TIME
|
132
|
+
remaining = deadline - ::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
|
133
|
+
end
|
134
|
+
end
|
136
135
|
end
|
137
136
|
end
|