sidekiq 4.2.4 → 6.4.0
Sign up to get free protection for your applications and to get access to all the features.
Potentially problematic release.
This version of sidekiq might be problematic. Click here for more details.
- checksums.yaml +5 -5
- data/Changes.md +523 -0
- data/LICENSE +3 -3
- data/README.md +23 -36
- data/bin/sidekiq +26 -2
- data/bin/sidekiqload +28 -38
- data/bin/sidekiqmon +8 -0
- data/lib/generators/sidekiq/job_generator.rb +57 -0
- data/lib/generators/sidekiq/templates/{worker.rb.erb → job.rb.erb} +2 -2
- data/lib/generators/sidekiq/templates/job_spec.rb.erb +6 -0
- data/lib/generators/sidekiq/templates/job_test.rb.erb +8 -0
- data/lib/sidekiq/api.rb +403 -243
- data/lib/sidekiq/cli.rb +230 -211
- data/lib/sidekiq/client.rb +53 -64
- data/lib/sidekiq/delay.rb +43 -0
- data/lib/sidekiq/exception_handler.rb +12 -16
- data/lib/sidekiq/extensions/action_mailer.rb +15 -24
- data/lib/sidekiq/extensions/active_record.rb +15 -12
- data/lib/sidekiq/extensions/class_methods.rb +16 -13
- data/lib/sidekiq/extensions/generic_proxy.rb +14 -6
- data/lib/sidekiq/fetch.rb +39 -31
- data/lib/sidekiq/job.rb +13 -0
- data/lib/sidekiq/job_logger.rb +63 -0
- data/lib/sidekiq/job_retry.rb +261 -0
- data/lib/sidekiq/job_util.rb +65 -0
- data/lib/sidekiq/launcher.rb +170 -71
- data/lib/sidekiq/logger.rb +166 -0
- data/lib/sidekiq/manager.rb +21 -26
- data/lib/sidekiq/middleware/chain.rb +20 -8
- data/lib/sidekiq/middleware/current_attributes.rb +57 -0
- data/lib/sidekiq/middleware/i18n.rb +5 -7
- data/lib/sidekiq/monitor.rb +133 -0
- data/lib/sidekiq/paginator.rb +18 -14
- data/lib/sidekiq/processor.rb +161 -70
- data/lib/sidekiq/rails.rb +41 -73
- data/lib/sidekiq/redis_connection.rb +65 -20
- data/lib/sidekiq/scheduled.rb +95 -34
- data/lib/sidekiq/sd_notify.rb +149 -0
- data/lib/sidekiq/systemd.rb +24 -0
- data/lib/sidekiq/testing/inline.rb +2 -1
- data/lib/sidekiq/testing.rb +52 -26
- data/lib/sidekiq/util.rb +60 -14
- data/lib/sidekiq/version.rb +2 -1
- data/lib/sidekiq/web/action.rb +15 -15
- data/lib/sidekiq/web/application.rb +115 -89
- data/lib/sidekiq/web/csrf_protection.rb +180 -0
- data/lib/sidekiq/web/helpers.rb +151 -83
- data/lib/sidekiq/web/router.rb +27 -19
- data/lib/sidekiq/web.rb +65 -109
- data/lib/sidekiq/worker.rb +284 -41
- data/lib/sidekiq.rb +93 -60
- data/sidekiq.gemspec +24 -22
- data/web/assets/images/apple-touch-icon.png +0 -0
- data/web/assets/javascripts/application.js +83 -64
- data/web/assets/javascripts/dashboard.js +81 -85
- data/web/assets/stylesheets/application-dark.css +143 -0
- data/web/assets/stylesheets/application-rtl.css +242 -0
- data/web/assets/stylesheets/application.css +319 -143
- data/web/assets/stylesheets/bootstrap-rtl.min.css +9 -0
- data/web/assets/stylesheets/bootstrap.css +2 -2
- data/web/locales/ar.yml +87 -0
- data/web/locales/de.yml +14 -2
- data/web/locales/en.yml +8 -1
- data/web/locales/es.yml +22 -5
- data/web/locales/fa.yml +80 -0
- data/web/locales/fr.yml +10 -3
- data/web/locales/he.yml +79 -0
- data/web/locales/ja.yml +12 -4
- data/web/locales/lt.yml +83 -0
- data/web/locales/pl.yml +4 -4
- data/web/locales/ru.yml +4 -0
- data/web/locales/ur.yml +80 -0
- data/web/locales/vi.yml +83 -0
- data/web/views/_footer.erb +5 -2
- data/web/views/_job_info.erb +4 -3
- data/web/views/_nav.erb +4 -18
- data/web/views/_paging.erb +1 -1
- data/web/views/_poll_link.erb +2 -5
- data/web/views/_summary.erb +7 -7
- data/web/views/busy.erb +60 -22
- data/web/views/dashboard.erb +23 -15
- data/web/views/dead.erb +3 -3
- data/web/views/layout.erb +14 -3
- data/web/views/morgue.erb +19 -12
- data/web/views/queue.erb +24 -14
- data/web/views/queues.erb +14 -4
- data/web/views/retries.erb +22 -13
- data/web/views/retry.erb +4 -4
- data/web/views/scheduled.erb +7 -4
- metadata +49 -198
- data/.github/contributing.md +0 -32
- data/.github/issue_template.md +0 -4
- data/.gitignore +0 -12
- data/.travis.yml +0 -12
- data/3.0-Upgrade.md +0 -70
- data/4.0-Upgrade.md +0 -53
- data/COMM-LICENSE +0 -95
- data/Ent-Changes.md +0 -146
- data/Gemfile +0 -29
- data/Pro-2.0-Upgrade.md +0 -138
- data/Pro-3.0-Upgrade.md +0 -44
- data/Pro-Changes.md +0 -585
- data/Rakefile +0 -9
- data/bin/sidekiqctl +0 -99
- data/code_of_conduct.md +0 -50
- data/lib/generators/sidekiq/templates/worker_spec.rb.erb +0 -6
- data/lib/generators/sidekiq/templates/worker_test.rb.erb +0 -8
- data/lib/generators/sidekiq/worker_generator.rb +0 -49
- data/lib/sidekiq/core_ext.rb +0 -106
- data/lib/sidekiq/logging.rb +0 -106
- data/lib/sidekiq/middleware/server/active_record.rb +0 -13
- data/lib/sidekiq/middleware/server/logging.rb +0 -40
- data/lib/sidekiq/middleware/server/retry_jobs.rb +0 -205
- data/test/config.yml +0 -9
- data/test/env_based_config.yml +0 -11
- data/test/fake_env.rb +0 -1
- data/test/fixtures/en.yml +0 -2
- data/test/helper.rb +0 -75
- data/test/test_actors.rb +0 -138
- data/test/test_api.rb +0 -528
- data/test/test_cli.rb +0 -418
- data/test/test_client.rb +0 -266
- data/test/test_exception_handler.rb +0 -56
- data/test/test_extensions.rb +0 -127
- data/test/test_fetch.rb +0 -50
- data/test/test_launcher.rb +0 -95
- data/test/test_logging.rb +0 -35
- data/test/test_manager.rb +0 -50
- data/test/test_middleware.rb +0 -158
- data/test/test_processor.rb +0 -235
- data/test/test_rails.rb +0 -22
- data/test/test_redis_connection.rb +0 -132
- data/test/test_retry.rb +0 -326
- data/test/test_retry_exhausted.rb +0 -149
- data/test/test_scheduled.rb +0 -115
- data/test/test_scheduling.rb +0 -58
- data/test/test_sidekiq.rb +0 -107
- data/test/test_testing.rb +0 -143
- data/test/test_testing_fake.rb +0 -357
- data/test/test_testing_inline.rb +0 -94
- data/test/test_util.rb +0 -13
- data/test/test_web.rb +0 -726
- data/test/test_web_helpers.rb +0 -54
data/lib/sidekiq/launcher.rb
CHANGED
@@ -1,20 +1,28 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
|
-
|
3
|
-
require
|
4
|
-
require
|
5
|
-
require
|
2
|
+
|
3
|
+
require "sidekiq/manager"
|
4
|
+
require "sidekiq/fetch"
|
5
|
+
require "sidekiq/scheduled"
|
6
6
|
|
7
7
|
module Sidekiq
|
8
|
-
# The Launcher
|
9
|
-
# start, monitor and stop the core Actors in Sidekiq.
|
10
|
-
# If any of these actors die, the Sidekiq process exits
|
11
|
-
# immediately.
|
8
|
+
# The Launcher starts the Manager and Poller threads and provides the process heartbeat.
|
12
9
|
class Launcher
|
13
10
|
include Util
|
14
11
|
|
12
|
+
STATS_TTL = 5 * 365 * 24 * 60 * 60 # 5 years
|
13
|
+
|
14
|
+
PROCTITLES = [
|
15
|
+
proc { "sidekiq" },
|
16
|
+
proc { Sidekiq::VERSION },
|
17
|
+
proc { |me, data| data["tag"] },
|
18
|
+
proc { |me, data| "[#{Processor::WORKER_STATE.size} of #{data["concurrency"]} busy]" },
|
19
|
+
proc { |me, data| "stopping" if me.stopping? }
|
20
|
+
]
|
21
|
+
|
15
22
|
attr_accessor :manager, :poller, :fetcher
|
16
23
|
|
17
24
|
def initialize(options)
|
25
|
+
options[:fetch] ||= BasicFetch.new(options)
|
18
26
|
@manager = Sidekiq::Manager.new(options)
|
19
27
|
@poller = Sidekiq::Scheduled::Poller.new
|
20
28
|
@done = false
|
@@ -39,7 +47,7 @@ module Sidekiq
|
|
39
47
|
# return until all work is complete and cleaned up.
|
40
48
|
# It can take up to the timeout to complete.
|
41
49
|
def stop
|
42
|
-
deadline =
|
50
|
+
deadline = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC) + @options[:timeout]
|
43
51
|
|
44
52
|
@done = true
|
45
53
|
@manager.quiet
|
@@ -49,7 +57,7 @@ module Sidekiq
|
|
49
57
|
|
50
58
|
# Requeue everything in case there was a worker who grabbed work while stopped
|
51
59
|
# This call is a no-op in Sidekiq but necessary for Sidekiq Pro.
|
52
|
-
strategy =
|
60
|
+
strategy = @options[:fetch]
|
53
61
|
strategy.bulk_requeue([], @options)
|
54
62
|
|
55
63
|
clear_heartbeat
|
@@ -61,104 +69,195 @@ module Sidekiq
|
|
61
69
|
|
62
70
|
private unless $TESTING
|
63
71
|
|
64
|
-
|
72
|
+
BEAT_PAUSE = 5
|
73
|
+
|
74
|
+
def start_heartbeat
|
75
|
+
loop do
|
76
|
+
heartbeat
|
77
|
+
sleep BEAT_PAUSE
|
78
|
+
end
|
79
|
+
Sidekiq.logger.info("Heartbeat stopping...")
|
80
|
+
end
|
81
|
+
|
82
|
+
def clear_heartbeat
|
83
|
+
# Remove record from Redis since we are shutting down.
|
84
|
+
# Note we don't stop the heartbeat thread; if the process
|
85
|
+
# doesn't actually exit, it'll reappear in the Web UI.
|
86
|
+
Sidekiq.redis do |conn|
|
87
|
+
conn.pipelined do
|
88
|
+
conn.srem("processes", identity)
|
89
|
+
conn.unlink("#{identity}:workers")
|
90
|
+
end
|
91
|
+
end
|
92
|
+
rescue
|
93
|
+
# best effort, ignore network errors
|
94
|
+
end
|
95
|
+
|
96
|
+
def heartbeat
|
97
|
+
$0 = PROCTITLES.map { |proc| proc.call(self, to_data) }.compact.join(" ")
|
98
|
+
|
99
|
+
❤
|
100
|
+
end
|
101
|
+
|
102
|
+
def self.flush_stats
|
103
|
+
fails = Processor::FAILURE.reset
|
104
|
+
procd = Processor::PROCESSED.reset
|
105
|
+
return if fails + procd == 0
|
65
106
|
|
66
|
-
|
67
|
-
|
68
|
-
|
69
|
-
|
107
|
+
nowdate = Time.now.utc.strftime("%Y-%m-%d")
|
108
|
+
begin
|
109
|
+
Sidekiq.redis do |conn|
|
110
|
+
conn.pipelined do
|
111
|
+
conn.incrby("stat:processed", procd)
|
112
|
+
conn.incrby("stat:processed:#{nowdate}", procd)
|
113
|
+
conn.expire("stat:processed:#{nowdate}", STATS_TTL)
|
70
114
|
|
71
|
-
|
115
|
+
conn.incrby("stat:failed", fails)
|
116
|
+
conn.incrby("stat:failed:#{nowdate}", fails)
|
117
|
+
conn.expire("stat:failed:#{nowdate}", STATS_TTL)
|
118
|
+
end
|
119
|
+
end
|
120
|
+
rescue => ex
|
121
|
+
# we're exiting the process, things might be shut down so don't
|
122
|
+
# try to handle the exception
|
123
|
+
Sidekiq.logger.warn("Unable to flush stats: #{ex}")
|
124
|
+
end
|
72
125
|
end
|
126
|
+
at_exit(&method(:flush_stats))
|
73
127
|
|
74
|
-
def ❤
|
128
|
+
def ❤
|
129
|
+
key = identity
|
75
130
|
fails = procd = 0
|
131
|
+
|
76
132
|
begin
|
77
|
-
Processor::FAILURE.
|
78
|
-
Processor::PROCESSED.
|
133
|
+
fails = Processor::FAILURE.reset
|
134
|
+
procd = Processor::PROCESSED.reset
|
135
|
+
curstate = Processor::WORKER_STATE.dup
|
136
|
+
|
137
|
+
workers_key = "#{key}:workers"
|
138
|
+
nowdate = Time.now.utc.strftime("%Y-%m-%d")
|
79
139
|
|
80
|
-
workers_key = "#{key}:workers".freeze
|
81
|
-
nowdate = Time.now.utc.strftime("%Y-%m-%d".freeze)
|
82
140
|
Sidekiq.redis do |conn|
|
83
141
|
conn.multi do
|
84
|
-
conn.incrby("stat:processed"
|
142
|
+
conn.incrby("stat:processed", procd)
|
85
143
|
conn.incrby("stat:processed:#{nowdate}", procd)
|
86
|
-
conn.
|
144
|
+
conn.expire("stat:processed:#{nowdate}", STATS_TTL)
|
145
|
+
|
146
|
+
conn.incrby("stat:failed", fails)
|
87
147
|
conn.incrby("stat:failed:#{nowdate}", fails)
|
88
|
-
conn.
|
89
|
-
|
148
|
+
conn.expire("stat:failed:#{nowdate}", STATS_TTL)
|
149
|
+
|
150
|
+
conn.unlink(workers_key)
|
151
|
+
curstate.each_pair do |tid, hash|
|
90
152
|
conn.hset(workers_key, tid, Sidekiq.dump_json(hash))
|
91
153
|
end
|
92
154
|
conn.expire(workers_key, 60)
|
93
155
|
end
|
94
156
|
end
|
157
|
+
|
158
|
+
rtt = check_rtt
|
159
|
+
|
95
160
|
fails = procd = 0
|
161
|
+
kb = memory_usage(::Process.pid)
|
96
162
|
|
97
|
-
_, exists, _, _, msg = Sidekiq.redis
|
98
|
-
conn.multi
|
99
|
-
conn.sadd(
|
100
|
-
conn.exists(key)
|
101
|
-
conn.hmset(key,
|
163
|
+
_, exists, _, _, msg = Sidekiq.redis { |conn|
|
164
|
+
conn.multi {
|
165
|
+
conn.sadd("processes", key)
|
166
|
+
conn.exists?(key)
|
167
|
+
conn.hmset(key, "info", to_json,
|
168
|
+
"busy", curstate.size,
|
169
|
+
"beat", Time.now.to_f,
|
170
|
+
"rtt_us", rtt,
|
171
|
+
"quiet", @done,
|
172
|
+
"rss", kb)
|
102
173
|
conn.expire(key, 60)
|
103
174
|
conn.rpop("#{key}-signals")
|
104
|
-
|
105
|
-
|
175
|
+
}
|
176
|
+
}
|
106
177
|
|
107
178
|
# first heartbeat or recovering from an outage and need to reestablish our heartbeat
|
108
|
-
fire_event(:heartbeat)
|
179
|
+
fire_event(:heartbeat) unless exists
|
109
180
|
|
110
181
|
return unless msg
|
111
182
|
|
112
|
-
|
113
|
-
Sidekiq::CLI.instance.handle_signal(msg)
|
114
|
-
else
|
115
|
-
::Process.kill(msg, $$)
|
116
|
-
end
|
183
|
+
::Process.kill(msg, ::Process.pid)
|
117
184
|
rescue => e
|
118
185
|
# ignore all redis/network issues
|
119
|
-
logger.error("heartbeat: #{e
|
186
|
+
logger.error("heartbeat: #{e}")
|
120
187
|
# don't lose the counts if there was a network issue
|
121
|
-
Processor::PROCESSED.
|
122
|
-
Processor::FAILURE.
|
188
|
+
Processor::PROCESSED.incr(procd)
|
189
|
+
Processor::FAILURE.incr(fails)
|
123
190
|
end
|
124
191
|
end
|
125
192
|
|
126
|
-
|
127
|
-
|
128
|
-
|
129
|
-
|
130
|
-
|
131
|
-
'pid' => $$,
|
132
|
-
'tag' => @options[:tag] || '',
|
133
|
-
'concurrency' => @options[:concurrency],
|
134
|
-
'queues' => @options[:queues].uniq,
|
135
|
-
'labels' => @options[:labels],
|
136
|
-
'identity' => k,
|
137
|
-
}
|
138
|
-
# this data doesn't change so dump it to a string
|
139
|
-
# now so we don't need to dump it every heartbeat.
|
140
|
-
json = Sidekiq.dump_json(data)
|
193
|
+
# We run the heartbeat every five seconds.
|
194
|
+
# Capture five samples of RTT, log a warning if each sample
|
195
|
+
# is above our warning threshold.
|
196
|
+
RTT_READINGS = RingBuffer.new(5)
|
197
|
+
RTT_WARNING_LEVEL = 50_000
|
141
198
|
|
142
|
-
|
143
|
-
|
144
|
-
|
199
|
+
def check_rtt
|
200
|
+
a = b = 0
|
201
|
+
Sidekiq.redis do |x|
|
202
|
+
a = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC, :microsecond)
|
203
|
+
x.ping
|
204
|
+
b = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC, :microsecond)
|
145
205
|
end
|
146
|
-
|
206
|
+
rtt = b - a
|
207
|
+
RTT_READINGS << rtt
|
208
|
+
# Ideal RTT for Redis is < 1000µs
|
209
|
+
# Workable is < 10,000µs
|
210
|
+
# Log a warning if it's a disaster.
|
211
|
+
if RTT_READINGS.all? { |x| x > RTT_WARNING_LEVEL }
|
212
|
+
Sidekiq.logger.warn <<~EOM
|
213
|
+
Your Redis network connection is performing extremely poorly.
|
214
|
+
Last RTT readings were #{RTT_READINGS.buffer.inspect}, ideally these should be < 1000.
|
215
|
+
Ensure Redis is running in the same AZ or datacenter as Sidekiq.
|
216
|
+
If these values are close to 100,000, that means your Sidekiq process may be
|
217
|
+
CPU overloaded; see https://github.com/mperham/sidekiq/discussions/5039
|
218
|
+
EOM
|
219
|
+
RTT_READINGS.reset
|
220
|
+
end
|
221
|
+
rtt
|
147
222
|
end
|
148
223
|
|
149
|
-
|
150
|
-
|
151
|
-
|
152
|
-
|
153
|
-
|
154
|
-
|
155
|
-
conn.srem('processes', identity)
|
156
|
-
conn.del("#{identity}:workers")
|
224
|
+
MEMORY_GRABBER = case RUBY_PLATFORM
|
225
|
+
when /linux/
|
226
|
+
->(pid) {
|
227
|
+
IO.readlines("/proc/#{$$}/status").each do |line|
|
228
|
+
next unless line.start_with?("VmRSS:")
|
229
|
+
break line.split[1].to_i
|
157
230
|
end
|
158
|
-
|
159
|
-
|
160
|
-
|
231
|
+
}
|
232
|
+
when /darwin|bsd/
|
233
|
+
->(pid) {
|
234
|
+
`ps -o pid,rss -p #{pid}`.lines.last.split.last.to_i
|
235
|
+
}
|
236
|
+
else
|
237
|
+
->(pid) { 0 }
|
238
|
+
end
|
239
|
+
|
240
|
+
def memory_usage(pid)
|
241
|
+
MEMORY_GRABBER.call(pid)
|
242
|
+
end
|
243
|
+
|
244
|
+
def to_data
|
245
|
+
@data ||= {
|
246
|
+
"hostname" => hostname,
|
247
|
+
"started_at" => Time.now.to_f,
|
248
|
+
"pid" => ::Process.pid,
|
249
|
+
"tag" => @options[:tag] || "",
|
250
|
+
"concurrency" => @options[:concurrency],
|
251
|
+
"queues" => @options[:queues].uniq,
|
252
|
+
"labels" => @options[:labels],
|
253
|
+
"identity" => identity
|
254
|
+
}
|
161
255
|
end
|
162
256
|
|
257
|
+
def to_json
|
258
|
+
# this data changes infrequently so dump it to a string
|
259
|
+
# now so we don't need to dump it every heartbeat.
|
260
|
+
@json ||= Sidekiq.dump_json(to_data)
|
261
|
+
end
|
163
262
|
end
|
164
263
|
end
|
@@ -0,0 +1,166 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require "logger"
|
4
|
+
require "time"
|
5
|
+
|
6
|
+
module Sidekiq
|
7
|
+
module Context
|
8
|
+
def self.with(hash)
|
9
|
+
orig_context = current.dup
|
10
|
+
current.merge!(hash)
|
11
|
+
yield
|
12
|
+
ensure
|
13
|
+
Thread.current[:sidekiq_context] = orig_context
|
14
|
+
end
|
15
|
+
|
16
|
+
def self.current
|
17
|
+
Thread.current[:sidekiq_context] ||= {}
|
18
|
+
end
|
19
|
+
end
|
20
|
+
|
21
|
+
module LoggingUtils
|
22
|
+
LEVELS = {
|
23
|
+
"debug" => 0,
|
24
|
+
"info" => 1,
|
25
|
+
"warn" => 2,
|
26
|
+
"error" => 3,
|
27
|
+
"fatal" => 4
|
28
|
+
}
|
29
|
+
LEVELS.default_proc = proc do |_, level|
|
30
|
+
Sidekiq.logger.warn("Invalid log level: #{level.inspect}")
|
31
|
+
nil
|
32
|
+
end
|
33
|
+
|
34
|
+
def debug?
|
35
|
+
level <= 0
|
36
|
+
end
|
37
|
+
|
38
|
+
def info?
|
39
|
+
level <= 1
|
40
|
+
end
|
41
|
+
|
42
|
+
def warn?
|
43
|
+
level <= 2
|
44
|
+
end
|
45
|
+
|
46
|
+
def error?
|
47
|
+
level <= 3
|
48
|
+
end
|
49
|
+
|
50
|
+
def fatal?
|
51
|
+
level <= 4
|
52
|
+
end
|
53
|
+
|
54
|
+
def local_level
|
55
|
+
Thread.current[:sidekiq_log_level]
|
56
|
+
end
|
57
|
+
|
58
|
+
def local_level=(level)
|
59
|
+
case level
|
60
|
+
when Integer
|
61
|
+
Thread.current[:sidekiq_log_level] = level
|
62
|
+
when Symbol, String
|
63
|
+
Thread.current[:sidekiq_log_level] = LEVELS[level.to_s]
|
64
|
+
when nil
|
65
|
+
Thread.current[:sidekiq_log_level] = nil
|
66
|
+
else
|
67
|
+
raise ArgumentError, "Invalid log level: #{level.inspect}"
|
68
|
+
end
|
69
|
+
end
|
70
|
+
|
71
|
+
def level
|
72
|
+
local_level || super
|
73
|
+
end
|
74
|
+
|
75
|
+
# Change the thread-local level for the duration of the given block.
|
76
|
+
def log_at(level)
|
77
|
+
old_local_level = local_level
|
78
|
+
self.local_level = level
|
79
|
+
yield
|
80
|
+
ensure
|
81
|
+
self.local_level = old_local_level
|
82
|
+
end
|
83
|
+
|
84
|
+
# Redefined to check severity against #level, and thus the thread-local level, rather than +@level+.
|
85
|
+
# FIXME: Remove when the minimum Ruby version supports overriding Logger#level.
|
86
|
+
def add(severity, message = nil, progname = nil, &block)
|
87
|
+
severity ||= ::Logger::UNKNOWN
|
88
|
+
progname ||= @progname
|
89
|
+
|
90
|
+
return true if @logdev.nil? || severity < level
|
91
|
+
|
92
|
+
if message.nil?
|
93
|
+
if block
|
94
|
+
message = yield
|
95
|
+
else
|
96
|
+
message = progname
|
97
|
+
progname = @progname
|
98
|
+
end
|
99
|
+
end
|
100
|
+
|
101
|
+
@logdev.write format_message(format_severity(severity), Time.now, progname, message)
|
102
|
+
end
|
103
|
+
end
|
104
|
+
|
105
|
+
class Logger < ::Logger
|
106
|
+
include LoggingUtils
|
107
|
+
|
108
|
+
def initialize(*args, **kwargs)
|
109
|
+
super
|
110
|
+
self.formatter = Sidekiq.log_formatter
|
111
|
+
end
|
112
|
+
|
113
|
+
module Formatters
|
114
|
+
class Base < ::Logger::Formatter
|
115
|
+
def tid
|
116
|
+
Thread.current["sidekiq_tid"] ||= (Thread.current.object_id ^ ::Process.pid).to_s(36)
|
117
|
+
end
|
118
|
+
|
119
|
+
def ctx
|
120
|
+
Sidekiq::Context.current
|
121
|
+
end
|
122
|
+
|
123
|
+
def format_context
|
124
|
+
if ctx.any?
|
125
|
+
" " + ctx.compact.map { |k, v|
|
126
|
+
case v
|
127
|
+
when Array
|
128
|
+
"#{k}=#{v.join(",")}"
|
129
|
+
else
|
130
|
+
"#{k}=#{v}"
|
131
|
+
end
|
132
|
+
}.join(" ")
|
133
|
+
end
|
134
|
+
end
|
135
|
+
end
|
136
|
+
|
137
|
+
class Pretty < Base
|
138
|
+
def call(severity, time, program_name, message)
|
139
|
+
"#{time.utc.iso8601(3)} pid=#{::Process.pid} tid=#{tid}#{format_context} #{severity}: #{message}\n"
|
140
|
+
end
|
141
|
+
end
|
142
|
+
|
143
|
+
class WithoutTimestamp < Pretty
|
144
|
+
def call(severity, time, program_name, message)
|
145
|
+
"pid=#{::Process.pid} tid=#{tid}#{format_context} #{severity}: #{message}\n"
|
146
|
+
end
|
147
|
+
end
|
148
|
+
|
149
|
+
class JSON < Base
|
150
|
+
def call(severity, time, program_name, message)
|
151
|
+
hash = {
|
152
|
+
ts: time.utc.iso8601(3),
|
153
|
+
pid: ::Process.pid,
|
154
|
+
tid: tid,
|
155
|
+
lvl: severity,
|
156
|
+
msg: message
|
157
|
+
}
|
158
|
+
c = ctx
|
159
|
+
hash["ctx"] = c unless c.empty?
|
160
|
+
|
161
|
+
Sidekiq.dump_json(hash) << "\n"
|
162
|
+
end
|
163
|
+
end
|
164
|
+
end
|
165
|
+
end
|
166
|
+
end
|
data/lib/sidekiq/manager.rb
CHANGED
@@ -1,16 +1,14 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
|
-
# encoding: utf-8
|
3
|
-
require 'sidekiq/util'
|
4
|
-
require 'sidekiq/processor'
|
5
|
-
require 'sidekiq/fetch'
|
6
|
-
require 'thread'
|
7
|
-
require 'set'
|
8
2
|
|
9
|
-
|
3
|
+
require "sidekiq/util"
|
4
|
+
require "sidekiq/processor"
|
5
|
+
require "sidekiq/fetch"
|
6
|
+
require "set"
|
10
7
|
|
8
|
+
module Sidekiq
|
11
9
|
##
|
12
10
|
# The Manager is the central coordination point in Sidekiq, controlling
|
13
|
-
# the lifecycle of the Processors
|
11
|
+
# the lifecycle of the Processors.
|
14
12
|
#
|
15
13
|
# Tasks:
|
16
14
|
#
|
@@ -28,16 +26,16 @@ module Sidekiq
|
|
28
26
|
attr_reader :workers
|
29
27
|
attr_reader :options
|
30
28
|
|
31
|
-
def initialize(options={})
|
29
|
+
def initialize(options = {})
|
32
30
|
logger.debug { options.inspect }
|
33
31
|
@options = options
|
34
|
-
@count = options[:concurrency] ||
|
32
|
+
@count = options[:concurrency] || 10
|
35
33
|
raise ArgumentError, "Concurrency of #{@count} is not supported" if @count < 1
|
36
34
|
|
37
35
|
@done = false
|
38
36
|
@workers = Set.new
|
39
37
|
@count.times do
|
40
|
-
@workers << Processor.new(self)
|
38
|
+
@workers << Processor.new(self, options)
|
41
39
|
end
|
42
40
|
@plock = Mutex.new
|
43
41
|
end
|
@@ -54,15 +52,12 @@ module Sidekiq
|
|
54
52
|
|
55
53
|
logger.info { "Terminating quiet workers" }
|
56
54
|
@workers.each { |x| x.terminate }
|
57
|
-
fire_event(:quiet, true)
|
55
|
+
fire_event(:quiet, reverse: true)
|
58
56
|
end
|
59
57
|
|
60
|
-
# hack for quicker development / testing environment #2774
|
61
|
-
PAUSE_TIME = STDOUT.tty? ? 0.1 : 0.5
|
62
|
-
|
63
58
|
def stop(deadline)
|
64
59
|
quiet
|
65
|
-
fire_event(:shutdown, true)
|
60
|
+
fire_event(:shutdown, reverse: true)
|
66
61
|
|
67
62
|
# some of the shutdown events can be async,
|
68
63
|
# we don't have any way to know when they're done but
|
@@ -71,12 +66,7 @@ module Sidekiq
|
|
71
66
|
return if @workers.empty?
|
72
67
|
|
73
68
|
logger.info { "Pausing to allow workers to finish..." }
|
74
|
-
|
75
|
-
while remaining > PAUSE_TIME
|
76
|
-
return if @workers.empty?
|
77
|
-
sleep PAUSE_TIME
|
78
|
-
remaining = deadline - Time.now
|
79
|
-
end
|
69
|
+
wait_for(deadline) { @workers.empty? }
|
80
70
|
return if @workers.empty?
|
81
71
|
|
82
72
|
hard_shutdown
|
@@ -92,7 +82,7 @@ module Sidekiq
|
|
92
82
|
@plock.synchronize do
|
93
83
|
@workers.delete(processor)
|
94
84
|
unless @done
|
95
|
-
p = Processor.new(self)
|
85
|
+
p = Processor.new(self, options)
|
96
86
|
@workers << p
|
97
87
|
p.start
|
98
88
|
end
|
@@ -114,7 +104,7 @@ module Sidekiq
|
|
114
104
|
end
|
115
105
|
|
116
106
|
if cleanup.size > 0
|
117
|
-
jobs = cleanup.map {|p| p.job }.compact
|
107
|
+
jobs = cleanup.map { |p| p.job }.compact
|
118
108
|
|
119
109
|
logger.warn { "Terminating #{cleanup.size} busy worker threads" }
|
120
110
|
logger.warn { "Work still in progress #{jobs.inspect}" }
|
@@ -125,14 +115,19 @@ module Sidekiq
|
|
125
115
|
# contract says that jobs are run AT LEAST once. Process termination
|
126
116
|
# is delayed until we're certain the jobs are back in Redis because
|
127
117
|
# it is worse to lose a job than to run it twice.
|
128
|
-
strategy =
|
118
|
+
strategy = @options[:fetch]
|
129
119
|
strategy.bulk_requeue(jobs, @options)
|
130
120
|
end
|
131
121
|
|
132
122
|
cleanup.each do |processor|
|
133
123
|
processor.kill
|
134
124
|
end
|
135
|
-
end
|
136
125
|
|
126
|
+
# when this method returns, we immediately call `exit` which may not give
|
127
|
+
# the remaining threads time to run `ensure` blocks, etc. We pause here up
|
128
|
+
# to 3 seconds to give threads a minimal amount of time to run `ensure` blocks.
|
129
|
+
deadline = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC) + 3
|
130
|
+
wait_for(deadline) { @workers.empty? }
|
131
|
+
end
|
137
132
|
end
|
138
133
|
end
|
@@ -1,4 +1,5 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
|
+
|
2
3
|
module Sidekiq
|
3
4
|
# Middleware is code configured to run before/after
|
4
5
|
# a message is processed. It is patterned after Rack
|
@@ -66,7 +67,6 @@ module Sidekiq
|
|
66
67
|
module Middleware
|
67
68
|
class Chain
|
68
69
|
include Enumerable
|
69
|
-
attr_reader :entries
|
70
70
|
|
71
71
|
def initialize_copy(copy)
|
72
72
|
copy.instance_variable_set(:@entries, entries.dup)
|
@@ -77,21 +77,25 @@ module Sidekiq
|
|
77
77
|
end
|
78
78
|
|
79
79
|
def initialize
|
80
|
-
@entries =
|
80
|
+
@entries = nil
|
81
81
|
yield self if block_given?
|
82
82
|
end
|
83
83
|
|
84
|
+
def entries
|
85
|
+
@entries ||= []
|
86
|
+
end
|
87
|
+
|
84
88
|
def remove(klass)
|
85
89
|
entries.delete_if { |entry| entry.klass == klass }
|
86
90
|
end
|
87
91
|
|
88
92
|
def add(klass, *args)
|
89
|
-
remove(klass)
|
93
|
+
remove(klass)
|
90
94
|
entries << Entry.new(klass, *args)
|
91
95
|
end
|
92
96
|
|
93
97
|
def prepend(klass, *args)
|
94
|
-
remove(klass)
|
98
|
+
remove(klass)
|
95
99
|
entries.insert(0, Entry.new(klass, *args))
|
96
100
|
end
|
97
101
|
|
@@ -106,13 +110,17 @@ module Sidekiq
|
|
106
110
|
i = entries.index { |entry| entry.klass == newklass }
|
107
111
|
new_entry = i.nil? ? Entry.new(newklass, *args) : entries.delete_at(i)
|
108
112
|
i = entries.index { |entry| entry.klass == oldklass } || entries.count - 1
|
109
|
-
entries.insert(i+1, new_entry)
|
113
|
+
entries.insert(i + 1, new_entry)
|
110
114
|
end
|
111
115
|
|
112
116
|
def exists?(klass)
|
113
117
|
any? { |entry| entry.klass == klass }
|
114
118
|
end
|
115
119
|
|
120
|
+
def empty?
|
121
|
+
@entries.nil? || @entries.empty?
|
122
|
+
end
|
123
|
+
|
116
124
|
def retrieve
|
117
125
|
map(&:make_new)
|
118
126
|
end
|
@@ -122,8 +130,10 @@ module Sidekiq
|
|
122
130
|
end
|
123
131
|
|
124
132
|
def invoke(*args)
|
125
|
-
|
126
|
-
|
133
|
+
return yield if empty?
|
134
|
+
|
135
|
+
chain = retrieve
|
136
|
+
traverse_chain = proc do
|
127
137
|
if chain.empty?
|
128
138
|
yield
|
129
139
|
else
|
@@ -134,12 +144,14 @@ module Sidekiq
|
|
134
144
|
end
|
135
145
|
end
|
136
146
|
|
147
|
+
private
|
148
|
+
|
137
149
|
class Entry
|
138
150
|
attr_reader :klass
|
139
151
|
|
140
152
|
def initialize(klass, *args)
|
141
153
|
@klass = klass
|
142
|
-
@args
|
154
|
+
@args = args
|
143
155
|
end
|
144
156
|
|
145
157
|
def make_new
|