sidekiq 5.1.1 → 6.5.9
Sign up to get free protection for your applications and to get access to all the features.
Potentially problematic release.
This version of sidekiq might be problematic. Click here for more details.
- checksums.yaml +5 -5
- data/Changes.md +507 -1
- data/LICENSE +3 -3
- data/README.md +24 -35
- data/bin/sidekiq +27 -3
- data/bin/sidekiqload +80 -68
- data/bin/sidekiqmon +8 -0
- data/lib/generators/sidekiq/job_generator.rb +57 -0
- data/lib/generators/sidekiq/templates/{worker.rb.erb → job.rb.erb} +2 -2
- data/lib/generators/sidekiq/templates/{worker_spec.rb.erb → job_spec.rb.erb} +1 -1
- data/lib/generators/sidekiq/templates/{worker_test.rb.erb → job_test.rb.erb} +1 -1
- data/lib/sidekiq/api.rb +537 -286
- data/lib/sidekiq/cli.rb +243 -240
- data/lib/sidekiq/client.rb +82 -85
- data/lib/sidekiq/component.rb +65 -0
- data/lib/sidekiq/delay.rb +9 -7
- data/lib/sidekiq/extensions/action_mailer.rb +13 -22
- data/lib/sidekiq/extensions/active_record.rb +13 -10
- data/lib/sidekiq/extensions/class_methods.rb +14 -11
- data/lib/sidekiq/extensions/generic_proxy.rb +7 -5
- data/lib/sidekiq/fetch.rb +50 -40
- data/lib/sidekiq/job.rb +13 -0
- data/lib/sidekiq/job_logger.rb +36 -9
- data/lib/sidekiq/job_retry.rb +143 -97
- data/lib/sidekiq/job_util.rb +71 -0
- data/lib/sidekiq/launcher.rb +185 -85
- data/lib/sidekiq/logger.rb +156 -0
- data/lib/sidekiq/manager.rb +41 -43
- data/lib/sidekiq/metrics/deploy.rb +47 -0
- data/lib/sidekiq/metrics/query.rb +153 -0
- data/lib/sidekiq/metrics/shared.rb +94 -0
- data/lib/sidekiq/metrics/tracking.rb +134 -0
- data/lib/sidekiq/middleware/chain.rb +102 -46
- data/lib/sidekiq/middleware/current_attributes.rb +63 -0
- data/lib/sidekiq/middleware/i18n.rb +7 -7
- data/lib/sidekiq/middleware/modules.rb +21 -0
- data/lib/sidekiq/monitor.rb +133 -0
- data/lib/sidekiq/paginator.rb +28 -16
- data/lib/sidekiq/processor.rb +156 -98
- data/lib/sidekiq/rails.rb +48 -42
- data/lib/sidekiq/redis_client_adapter.rb +154 -0
- data/lib/sidekiq/redis_connection.rb +109 -51
- data/lib/sidekiq/ring_buffer.rb +29 -0
- data/lib/sidekiq/scheduled.rb +133 -41
- data/lib/sidekiq/sd_notify.rb +149 -0
- data/lib/sidekiq/systemd.rb +24 -0
- data/lib/sidekiq/testing/inline.rb +6 -5
- data/lib/sidekiq/testing.rb +72 -62
- data/lib/sidekiq/transaction_aware_client.rb +45 -0
- data/lib/sidekiq/version.rb +2 -1
- data/lib/sidekiq/web/action.rb +15 -11
- data/lib/sidekiq/web/application.rb +127 -76
- data/lib/sidekiq/web/csrf_protection.rb +180 -0
- data/lib/sidekiq/web/helpers.rb +133 -96
- data/lib/sidekiq/web/router.rb +23 -19
- data/lib/sidekiq/web.rb +69 -109
- data/lib/sidekiq/worker.rb +268 -102
- data/lib/sidekiq.rb +175 -66
- data/sidekiq.gemspec +23 -23
- data/web/assets/images/apple-touch-icon.png +0 -0
- data/web/assets/javascripts/application.js +112 -61
- data/web/assets/javascripts/chart.min.js +13 -0
- data/web/assets/javascripts/chartjs-plugin-annotation.min.js +7 -0
- data/web/assets/javascripts/dashboard.js +65 -91
- data/web/assets/javascripts/graph.js +16 -0
- data/web/assets/javascripts/metrics.js +262 -0
- data/web/assets/stylesheets/application-dark.css +143 -0
- data/web/assets/stylesheets/application-rtl.css +0 -4
- data/web/assets/stylesheets/application.css +120 -232
- data/web/assets/stylesheets/bootstrap.css +2 -2
- data/web/locales/ar.yml +9 -2
- data/web/locales/de.yml +14 -2
- data/web/locales/el.yml +43 -19
- data/web/locales/en.yml +14 -1
- data/web/locales/es.yml +21 -5
- data/web/locales/fr.yml +10 -3
- data/web/locales/ja.yml +14 -1
- data/web/locales/lt.yml +83 -0
- data/web/locales/pl.yml +4 -4
- data/web/locales/pt-br.yml +27 -9
- data/web/locales/ru.yml +4 -0
- data/web/locales/vi.yml +83 -0
- data/web/locales/zh-cn.yml +36 -11
- data/web/locales/zh-tw.yml +32 -7
- data/web/views/_footer.erb +4 -1
- data/web/views/_job_info.erb +3 -2
- data/web/views/_nav.erb +4 -18
- data/web/views/_poll_link.erb +2 -5
- data/web/views/_summary.erb +7 -7
- data/web/views/busy.erb +61 -22
- data/web/views/dashboard.erb +23 -14
- data/web/views/dead.erb +3 -3
- data/web/views/layout.erb +4 -2
- data/web/views/metrics.erb +69 -0
- data/web/views/metrics_for_job.erb +87 -0
- data/web/views/morgue.erb +9 -6
- data/web/views/queue.erb +24 -10
- data/web/views/queues.erb +11 -3
- data/web/views/retries.erb +14 -7
- data/web/views/retry.erb +3 -3
- data/web/views/scheduled.erb +5 -2
- metadata +62 -135
- data/.github/contributing.md +0 -32
- data/.github/issue_template.md +0 -11
- data/.gitignore +0 -13
- data/.travis.yml +0 -14
- data/3.0-Upgrade.md +0 -70
- data/4.0-Upgrade.md +0 -53
- data/5.0-Upgrade.md +0 -56
- data/COMM-LICENSE +0 -95
- data/Ent-Changes.md +0 -210
- data/Gemfile +0 -8
- data/Pro-2.0-Upgrade.md +0 -138
- data/Pro-3.0-Upgrade.md +0 -44
- data/Pro-4.0-Upgrade.md +0 -35
- data/Pro-Changes.md +0 -716
- data/Rakefile +0 -8
- data/bin/sidekiqctl +0 -99
- data/code_of_conduct.md +0 -50
- data/lib/generators/sidekiq/worker_generator.rb +0 -49
- data/lib/sidekiq/core_ext.rb +0 -1
- data/lib/sidekiq/exception_handler.rb +0 -29
- data/lib/sidekiq/logging.rb +0 -122
- data/lib/sidekiq/middleware/server/active_record.rb +0 -22
- data/lib/sidekiq/middleware/server/active_record_cache.rb +0 -11
- data/lib/sidekiq/util.rb +0 -66
data/lib/sidekiq/api.rb
CHANGED
@@ -1,11 +1,36 @@
|
|
1
|
-
# encoding: utf-8
|
2
1
|
# frozen_string_literal: true
|
3
|
-
|
2
|
+
|
3
|
+
require "sidekiq"
|
4
|
+
|
5
|
+
require "zlib"
|
6
|
+
require "set"
|
7
|
+
require "base64"
|
8
|
+
|
9
|
+
if ENV["SIDEKIQ_METRICS_BETA"]
|
10
|
+
require "sidekiq/metrics/deploy"
|
11
|
+
require "sidekiq/metrics/query"
|
12
|
+
end
|
13
|
+
|
14
|
+
#
|
15
|
+
# Sidekiq's Data API provides a Ruby object model on top
|
16
|
+
# of Sidekiq's runtime data in Redis. This API should never
|
17
|
+
# be used within application code for business logic.
|
18
|
+
#
|
19
|
+
# The Sidekiq server process never uses this API: all data
|
20
|
+
# manipulation is done directly for performance reasons to
|
21
|
+
# ensure we are using Redis as efficiently as possible at
|
22
|
+
# every callsite.
|
23
|
+
#
|
4
24
|
|
5
25
|
module Sidekiq
|
26
|
+
# Retrieve runtime statistics from Redis regarding
|
27
|
+
# this Sidekiq cluster.
|
28
|
+
#
|
29
|
+
# stat = Sidekiq::Stats.new
|
30
|
+
# stat.processed
|
6
31
|
class Stats
|
7
32
|
def initialize
|
8
|
-
|
33
|
+
fetch_stats_fast!
|
9
34
|
end
|
10
35
|
|
11
36
|
def processed
|
@@ -48,56 +73,82 @@ module Sidekiq
|
|
48
73
|
Sidekiq::Stats::Queues.new.lengths
|
49
74
|
end
|
50
75
|
|
51
|
-
|
52
|
-
|
53
|
-
|
54
|
-
|
55
|
-
|
56
|
-
|
57
|
-
|
58
|
-
|
59
|
-
|
60
|
-
|
61
|
-
|
62
|
-
|
76
|
+
# O(1) redis calls
|
77
|
+
# @api private
|
78
|
+
def fetch_stats_fast!
|
79
|
+
pipe1_res = Sidekiq.redis { |conn|
|
80
|
+
conn.pipelined do |pipeline|
|
81
|
+
pipeline.get("stat:processed")
|
82
|
+
pipeline.get("stat:failed")
|
83
|
+
pipeline.zcard("schedule")
|
84
|
+
pipeline.zcard("retry")
|
85
|
+
pipeline.zcard("dead")
|
86
|
+
pipeline.scard("processes")
|
87
|
+
pipeline.lrange("queue:default", -1, -1)
|
63
88
|
end
|
64
|
-
|
89
|
+
}
|
65
90
|
|
66
|
-
|
67
|
-
|
68
|
-
|
69
|
-
|
91
|
+
default_queue_latency = if (entry = pipe1_res[6].first)
|
92
|
+
job = begin
|
93
|
+
Sidekiq.load_json(entry)
|
94
|
+
rescue
|
95
|
+
{}
|
70
96
|
end
|
97
|
+
now = Time.now.to_f
|
98
|
+
thence = job["enqueued_at"] || now
|
99
|
+
now - thence
|
100
|
+
else
|
101
|
+
0
|
71
102
|
end
|
72
103
|
|
73
|
-
s = pipe1_res[7].size
|
74
|
-
workers_size = pipe2_res[0...s].map(&:to_i).inject(0, &:+)
|
75
|
-
enqueued = pipe2_res[s..-1].map(&:to_i).inject(0, &:+)
|
76
|
-
|
77
|
-
default_queue_latency = if (entry = pipe1_res[6].first)
|
78
|
-
job = Sidekiq.load_json(entry) rescue {}
|
79
|
-
now = Time.now.to_f
|
80
|
-
thence = job['enqueued_at'.freeze] || now
|
81
|
-
now - thence
|
82
|
-
else
|
83
|
-
0
|
84
|
-
end
|
85
104
|
@stats = {
|
86
|
-
processed:
|
87
|
-
failed:
|
88
|
-
scheduled_size:
|
89
|
-
retry_size:
|
90
|
-
dead_size:
|
91
|
-
processes_size:
|
92
|
-
|
93
|
-
default_queue_latency: default_queue_latency
|
94
|
-
workers_size: workers_size,
|
95
|
-
enqueued: enqueued
|
105
|
+
processed: pipe1_res[0].to_i,
|
106
|
+
failed: pipe1_res[1].to_i,
|
107
|
+
scheduled_size: pipe1_res[2],
|
108
|
+
retry_size: pipe1_res[3],
|
109
|
+
dead_size: pipe1_res[4],
|
110
|
+
processes_size: pipe1_res[5],
|
111
|
+
|
112
|
+
default_queue_latency: default_queue_latency
|
96
113
|
}
|
97
114
|
end
|
98
115
|
|
116
|
+
# O(number of processes + number of queues) redis calls
|
117
|
+
# @api private
|
118
|
+
def fetch_stats_slow!
|
119
|
+
processes = Sidekiq.redis { |conn|
|
120
|
+
conn.sscan_each("processes").to_a
|
121
|
+
}
|
122
|
+
|
123
|
+
queues = Sidekiq.redis { |conn|
|
124
|
+
conn.sscan_each("queues").to_a
|
125
|
+
}
|
126
|
+
|
127
|
+
pipe2_res = Sidekiq.redis { |conn|
|
128
|
+
conn.pipelined do |pipeline|
|
129
|
+
processes.each { |key| pipeline.hget(key, "busy") }
|
130
|
+
queues.each { |queue| pipeline.llen("queue:#{queue}") }
|
131
|
+
end
|
132
|
+
}
|
133
|
+
|
134
|
+
s = processes.size
|
135
|
+
workers_size = pipe2_res[0...s].sum(&:to_i)
|
136
|
+
enqueued = pipe2_res[s..-1].sum(&:to_i)
|
137
|
+
|
138
|
+
@stats[:workers_size] = workers_size
|
139
|
+
@stats[:enqueued] = enqueued
|
140
|
+
@stats
|
141
|
+
end
|
142
|
+
|
143
|
+
# @api private
|
144
|
+
def fetch_stats!
|
145
|
+
fetch_stats_fast!
|
146
|
+
fetch_stats_slow!
|
147
|
+
end
|
148
|
+
|
149
|
+
# @api private
|
99
150
|
def reset(*stats)
|
100
|
-
all
|
151
|
+
all = %w[failed processed]
|
101
152
|
stats = stats.empty? ? all : all & stats.flatten.compact.map(&:to_s)
|
102
153
|
|
103
154
|
mset_args = []
|
@@ -113,34 +164,31 @@ module Sidekiq
|
|
113
164
|
private
|
114
165
|
|
115
166
|
def stat(s)
|
116
|
-
@stats[s]
|
167
|
+
fetch_stats_slow! if @stats[s].nil?
|
168
|
+
@stats[s] || raise(ArgumentError, "Unknown stat #{s}")
|
117
169
|
end
|
118
170
|
|
119
171
|
class Queues
|
120
172
|
def lengths
|
121
173
|
Sidekiq.redis do |conn|
|
122
|
-
queues = conn.
|
174
|
+
queues = conn.sscan_each("queues").to_a
|
123
175
|
|
124
|
-
lengths = conn.pipelined
|
176
|
+
lengths = conn.pipelined { |pipeline|
|
125
177
|
queues.each do |queue|
|
126
|
-
|
178
|
+
pipeline.llen("queue:#{queue}")
|
127
179
|
end
|
128
|
-
|
129
|
-
|
130
|
-
i = 0
|
131
|
-
array_of_arrays = queues.inject({}) do |memo, queue|
|
132
|
-
memo[queue] = lengths[i]
|
133
|
-
i += 1
|
134
|
-
memo
|
135
|
-
end.sort_by { |_, size| size }
|
180
|
+
}
|
136
181
|
|
137
|
-
|
182
|
+
array_of_arrays = queues.zip(lengths).sort_by { |_, size| -size }
|
183
|
+
array_of_arrays.to_h
|
138
184
|
end
|
139
185
|
end
|
140
186
|
end
|
141
187
|
|
142
188
|
class History
|
143
189
|
def initialize(days_previous, start_date = nil)
|
190
|
+
# we only store five years of data in Redis
|
191
|
+
raise ArgumentError if days_previous < 1 || days_previous > (5 * 365)
|
144
192
|
@days_previous = days_previous
|
145
193
|
@start_date = start_date || Time.now.utc.to_date
|
146
194
|
end
|
@@ -156,18 +204,12 @@ module Sidekiq
|
|
156
204
|
private
|
157
205
|
|
158
206
|
def date_stat_hash(stat)
|
159
|
-
i = 0
|
160
207
|
stat_hash = {}
|
161
|
-
|
162
|
-
|
163
|
-
|
164
|
-
|
165
|
-
|
166
|
-
datestr = date.strftime("%Y-%m-%d".freeze)
|
167
|
-
keys << "stat:#{stat}:#{datestr}"
|
168
|
-
dates << datestr
|
169
|
-
i += 1
|
170
|
-
end
|
208
|
+
dates = @start_date.downto(@start_date - @days_previous + 1).map { |date|
|
209
|
+
date.strftime("%Y-%m-%d")
|
210
|
+
}
|
211
|
+
|
212
|
+
keys = dates.map { |datestr| "stat:#{stat}:#{datestr}" }
|
171
213
|
|
172
214
|
begin
|
173
215
|
Sidekiq.redis do |conn|
|
@@ -175,7 +217,7 @@ module Sidekiq
|
|
175
217
|
stat_hash[dates[idx]] = value ? value.to_i : 0
|
176
218
|
end
|
177
219
|
end
|
178
|
-
rescue
|
220
|
+
rescue RedisConnection.adapter::CommandError
|
179
221
|
# mget will trigger a CROSSSLOT error when run against a Cluster
|
180
222
|
# TODO Someone want to add Cluster support?
|
181
223
|
end
|
@@ -186,9 +228,10 @@ module Sidekiq
|
|
186
228
|
end
|
187
229
|
|
188
230
|
##
|
189
|
-
#
|
231
|
+
# Represents a queue within Sidekiq.
|
190
232
|
# Allows enumeration of all jobs within the queue
|
191
|
-
# and deletion of jobs.
|
233
|
+
# and deletion of jobs. NB: this queue data is real-time
|
234
|
+
# and is changing within Redis moment by moment.
|
192
235
|
#
|
193
236
|
# queue = Sidekiq::Queue.new("mailer")
|
194
237
|
# queue.each do |job|
|
@@ -196,29 +239,34 @@ module Sidekiq
|
|
196
239
|
# job.args # => [1, 2, 3]
|
197
240
|
# job.delete if job.jid == 'abcdef1234567890'
|
198
241
|
# end
|
199
|
-
#
|
200
242
|
class Queue
|
201
243
|
include Enumerable
|
202
244
|
|
203
245
|
##
|
204
|
-
#
|
246
|
+
# Fetch all known queues within Redis.
|
205
247
|
#
|
248
|
+
# @return [Array<Sidekiq::Queue>]
|
206
249
|
def self.all
|
207
|
-
Sidekiq.redis { |c| c.
|
250
|
+
Sidekiq.redis { |c| c.sscan_each("queues").to_a }.sort.map { |q| Sidekiq::Queue.new(q) }
|
208
251
|
end
|
209
252
|
|
210
253
|
attr_reader :name
|
211
254
|
|
212
|
-
|
213
|
-
|
255
|
+
# @param name [String] the name of the queue
|
256
|
+
def initialize(name = "default")
|
257
|
+
@name = name.to_s
|
214
258
|
@rname = "queue:#{name}"
|
215
259
|
end
|
216
260
|
|
261
|
+
# The current size of the queue within Redis.
|
262
|
+
# This value is real-time and can change between calls.
|
263
|
+
#
|
264
|
+
# @return [Integer] the size
|
217
265
|
def size
|
218
266
|
Sidekiq.redis { |con| con.llen(@rname) }
|
219
267
|
end
|
220
268
|
|
221
|
-
#
|
269
|
+
# @return [Boolean] if the queue is currently paused
|
222
270
|
def paused?
|
223
271
|
false
|
224
272
|
end
|
@@ -227,15 +275,15 @@ module Sidekiq
|
|
227
275
|
# Calculates this queue's latency, the difference in seconds since the oldest
|
228
276
|
# job in the queue was enqueued.
|
229
277
|
#
|
230
|
-
# @return Float
|
278
|
+
# @return [Float] in seconds
|
231
279
|
def latency
|
232
|
-
entry = Sidekiq.redis
|
280
|
+
entry = Sidekiq.redis { |conn|
|
233
281
|
conn.lrange(@rname, -1, -1)
|
234
|
-
|
282
|
+
}.first
|
235
283
|
return 0 unless entry
|
236
284
|
job = Sidekiq.load_json(entry)
|
237
285
|
now = Time.now.to_f
|
238
|
-
thence = job[
|
286
|
+
thence = job["enqueued_at"] || now
|
239
287
|
now - thence
|
240
288
|
end
|
241
289
|
|
@@ -245,16 +293,16 @@ module Sidekiq
|
|
245
293
|
page = 0
|
246
294
|
page_size = 50
|
247
295
|
|
248
|
-
|
296
|
+
loop do
|
249
297
|
range_start = page * page_size - deleted_size
|
250
|
-
range_end
|
251
|
-
entries = Sidekiq.redis
|
298
|
+
range_end = range_start + page_size - 1
|
299
|
+
entries = Sidekiq.redis { |conn|
|
252
300
|
conn.lrange @rname, range_start, range_end
|
253
|
-
|
301
|
+
}
|
254
302
|
break if entries.empty?
|
255
303
|
page += 1
|
256
304
|
entries.each do |entry|
|
257
|
-
yield
|
305
|
+
yield JobRecord.new(entry, @name)
|
258
306
|
end
|
259
307
|
deleted_size = initial_size - size
|
260
308
|
end
|
@@ -263,41 +311,63 @@ module Sidekiq
|
|
263
311
|
##
|
264
312
|
# Find the job with the given JID within this queue.
|
265
313
|
#
|
266
|
-
# This is a slow, inefficient operation. Do not use under
|
267
|
-
# normal conditions.
|
314
|
+
# This is a *slow, inefficient* operation. Do not use under
|
315
|
+
# normal conditions.
|
316
|
+
#
|
317
|
+
# @param jid [String] the job_id to look for
|
318
|
+
# @return [Sidekiq::JobRecord]
|
319
|
+
# @return [nil] if not found
|
268
320
|
def find_job(jid)
|
269
321
|
detect { |j| j.jid == jid }
|
270
322
|
end
|
271
323
|
|
324
|
+
# delete all jobs within this queue
|
325
|
+
# @return [Boolean] true
|
272
326
|
def clear
|
273
327
|
Sidekiq.redis do |conn|
|
274
|
-
conn.multi do
|
275
|
-
|
276
|
-
|
328
|
+
conn.multi do |transaction|
|
329
|
+
transaction.unlink(@rname)
|
330
|
+
transaction.srem("queues", [name])
|
277
331
|
end
|
278
332
|
end
|
333
|
+
true
|
279
334
|
end
|
280
335
|
alias_method :💣, :clear
|
336
|
+
|
337
|
+
# :nodoc:
|
338
|
+
# @api private
|
339
|
+
def as_json(options = nil)
|
340
|
+
{name: name} # 5336
|
341
|
+
end
|
281
342
|
end
|
282
343
|
|
283
344
|
##
|
284
|
-
#
|
285
|
-
# sorted set.
|
345
|
+
# Represents a pending job within a Sidekiq queue.
|
286
346
|
#
|
287
347
|
# The job should be considered immutable but may be
|
288
|
-
# removed from the queue via
|
289
|
-
|
290
|
-
|
348
|
+
# removed from the queue via JobRecord#delete.
|
349
|
+
class JobRecord
|
350
|
+
# the parsed Hash of job data
|
351
|
+
# @!attribute [r] Item
|
291
352
|
attr_reader :item
|
353
|
+
# the underlying String in Redis
|
354
|
+
# @!attribute [r] Value
|
292
355
|
attr_reader :value
|
356
|
+
# the queue associated with this job
|
357
|
+
# @!attribute [r] Queue
|
358
|
+
attr_reader :queue
|
293
359
|
|
294
|
-
|
360
|
+
# :nodoc:
|
361
|
+
# @api private
|
362
|
+
def initialize(item, queue_name = nil)
|
295
363
|
@args = nil
|
296
364
|
@value = item
|
297
365
|
@item = item.is_a?(Hash) ? item : parse(item)
|
298
|
-
@queue = queue_name || @item[
|
366
|
+
@queue = queue_name || @item["queue"]
|
299
367
|
end
|
300
368
|
|
369
|
+
# :nodoc:
|
370
|
+
# @api private
|
301
371
|
def parse(item)
|
302
372
|
Sidekiq.load_json(item)
|
303
373
|
rescue JSON::ParserError
|
@@ -309,88 +379,109 @@ module Sidekiq
|
|
309
379
|
{}
|
310
380
|
end
|
311
381
|
|
382
|
+
# This is the job class which Sidekiq will execute. If using ActiveJob,
|
383
|
+
# this class will be the ActiveJob adapter class rather than a specific job.
|
312
384
|
def klass
|
313
|
-
self[
|
385
|
+
self["class"]
|
314
386
|
end
|
315
387
|
|
316
388
|
def display_class
|
317
389
|
# Unwrap known wrappers so they show up in a human-friendly manner in the Web UI
|
318
|
-
@klass ||=
|
319
|
-
|
320
|
-
|
321
|
-
|
322
|
-
|
323
|
-
|
324
|
-
|
325
|
-
|
326
|
-
|
327
|
-
|
328
|
-
|
329
|
-
|
330
|
-
|
331
|
-
|
332
|
-
|
333
|
-
|
390
|
+
@klass ||= self["display_class"] || begin
|
391
|
+
case klass
|
392
|
+
when /\ASidekiq::Extensions::Delayed/
|
393
|
+
safe_load(args[0], klass) do |target, method, _|
|
394
|
+
"#{target}.#{method}"
|
395
|
+
end
|
396
|
+
when "ActiveJob::QueueAdapters::SidekiqAdapter::JobWrapper"
|
397
|
+
job_class = @item["wrapped"] || args[0]
|
398
|
+
if job_class == "ActionMailer::DeliveryJob" || job_class == "ActionMailer::MailDeliveryJob"
|
399
|
+
# MailerClass#mailer_method
|
400
|
+
args[0]["arguments"][0..1].join("#")
|
401
|
+
else
|
402
|
+
job_class
|
403
|
+
end
|
404
|
+
else
|
405
|
+
klass
|
406
|
+
end
|
407
|
+
end
|
334
408
|
end
|
335
409
|
|
336
410
|
def display_args
|
337
411
|
# Unwrap known wrappers so they show up in a human-friendly manner in the Web UI
|
338
412
|
@display_args ||= case klass
|
339
|
-
|
340
|
-
|
341
|
-
|
342
|
-
|
343
|
-
|
344
|
-
|
345
|
-
|
346
|
-
|
347
|
-
|
348
|
-
|
349
|
-
|
350
|
-
|
351
|
-
|
352
|
-
|
353
|
-
|
354
|
-
|
355
|
-
|
356
|
-
|
357
|
-
|
413
|
+
when /\ASidekiq::Extensions::Delayed/
|
414
|
+
safe_load(args[0], args) do |_, _, arg, kwarg|
|
415
|
+
if !kwarg || kwarg.empty?
|
416
|
+
arg
|
417
|
+
else
|
418
|
+
[arg, kwarg]
|
419
|
+
end
|
420
|
+
end
|
421
|
+
when "ActiveJob::QueueAdapters::SidekiqAdapter::JobWrapper"
|
422
|
+
job_args = self["wrapped"] ? args[0]["arguments"] : []
|
423
|
+
if (self["wrapped"] || args[0]) == "ActionMailer::DeliveryJob"
|
424
|
+
# remove MailerClass, mailer_method and 'deliver_now'
|
425
|
+
job_args.drop(3)
|
426
|
+
elsif (self["wrapped"] || args[0]) == "ActionMailer::MailDeliveryJob"
|
427
|
+
# remove MailerClass, mailer_method and 'deliver_now'
|
428
|
+
job_args.drop(3).first["args"]
|
429
|
+
else
|
430
|
+
job_args
|
431
|
+
end
|
432
|
+
else
|
433
|
+
if self["encrypt"]
|
434
|
+
# no point in showing 150+ bytes of random garbage
|
435
|
+
args[-1] = "[encrypted data]"
|
436
|
+
end
|
437
|
+
args
|
438
|
+
end
|
358
439
|
end
|
359
440
|
|
360
441
|
def args
|
361
|
-
@args || @item[
|
442
|
+
@args || @item["args"]
|
362
443
|
end
|
363
444
|
|
364
445
|
def jid
|
365
|
-
self[
|
446
|
+
self["jid"]
|
366
447
|
end
|
367
448
|
|
368
449
|
def enqueued_at
|
369
|
-
self[
|
450
|
+
self["enqueued_at"] ? Time.at(self["enqueued_at"]).utc : nil
|
370
451
|
end
|
371
452
|
|
372
453
|
def created_at
|
373
|
-
Time.at(self[
|
454
|
+
Time.at(self["created_at"] || self["enqueued_at"] || 0).utc
|
374
455
|
end
|
375
456
|
|
376
|
-
def
|
377
|
-
|
457
|
+
def tags
|
458
|
+
self["tags"] || []
|
459
|
+
end
|
460
|
+
|
461
|
+
def error_backtrace
|
462
|
+
# Cache nil values
|
463
|
+
if defined?(@error_backtrace)
|
464
|
+
@error_backtrace
|
465
|
+
else
|
466
|
+
value = self["error_backtrace"]
|
467
|
+
@error_backtrace = value && uncompress_backtrace(value)
|
468
|
+
end
|
378
469
|
end
|
379
470
|
|
380
471
|
def latency
|
381
472
|
now = Time.now.to_f
|
382
|
-
now - (@item[
|
473
|
+
now - (@item["enqueued_at"] || @item["created_at"] || now)
|
383
474
|
end
|
384
475
|
|
385
|
-
|
386
|
-
# Remove this job from the queue.
|
476
|
+
# Remove this job from the queue
|
387
477
|
def delete
|
388
|
-
count = Sidekiq.redis
|
478
|
+
count = Sidekiq.redis { |conn|
|
389
479
|
conn.lrem("queue:#{@queue}", 1, @value)
|
390
|
-
|
480
|
+
}
|
391
481
|
count != 0
|
392
482
|
end
|
393
483
|
|
484
|
+
# Access arbitrary attributes within the job hash
|
394
485
|
def [](name)
|
395
486
|
# nil will happen if the JSON fails to parse.
|
396
487
|
# We don't guarantee Sidekiq will work with bad job JSON but we should
|
@@ -401,31 +492,55 @@ module Sidekiq
|
|
401
492
|
private
|
402
493
|
|
403
494
|
def safe_load(content, default)
|
404
|
-
|
405
|
-
|
406
|
-
|
407
|
-
|
408
|
-
|
409
|
-
|
410
|
-
|
495
|
+
yield(*YAML.load(content))
|
496
|
+
rescue => ex
|
497
|
+
# #1761 in dev mode, it's possible to have jobs enqueued which haven't been loaded into
|
498
|
+
# memory yet so the YAML can't be loaded.
|
499
|
+
# TODO is this still necessary? Zeitwerk reloader should handle?
|
500
|
+
Sidekiq.logger.warn "Unable to load YAML: #{ex.message}" unless Sidekiq.options[:environment] == "development"
|
501
|
+
default
|
502
|
+
end
|
503
|
+
|
504
|
+
def uncompress_backtrace(backtrace)
|
505
|
+
if backtrace.is_a?(Array)
|
506
|
+
# Handle old jobs with raw Array backtrace format
|
507
|
+
backtrace
|
508
|
+
else
|
509
|
+
decoded = Base64.decode64(backtrace)
|
510
|
+
uncompressed = Zlib::Inflate.inflate(decoded)
|
511
|
+
begin
|
512
|
+
Sidekiq.load_json(uncompressed)
|
513
|
+
rescue
|
514
|
+
# Handle old jobs with marshalled backtrace format
|
515
|
+
# TODO Remove in 7.x
|
516
|
+
Marshal.load(uncompressed)
|
517
|
+
end
|
411
518
|
end
|
412
519
|
end
|
413
520
|
end
|
414
521
|
|
415
|
-
|
522
|
+
# Represents a job within a Redis sorted set where the score
|
523
|
+
# represents a timestamp associated with the job. This timestamp
|
524
|
+
# could be the scheduled time for it to run (e.g. scheduled set),
|
525
|
+
# or the expiration date after which the entry should be deleted (e.g. dead set).
|
526
|
+
class SortedEntry < JobRecord
|
416
527
|
attr_reader :score
|
417
528
|
attr_reader :parent
|
418
529
|
|
530
|
+
# :nodoc:
|
531
|
+
# @api private
|
419
532
|
def initialize(parent, score, item)
|
420
533
|
super(item)
|
421
|
-
@score = score
|
534
|
+
@score = Float(score)
|
422
535
|
@parent = parent
|
423
536
|
end
|
424
537
|
|
538
|
+
# The timestamp associated with this entry
|
425
539
|
def at
|
426
540
|
Time.at(score).utc
|
427
541
|
end
|
428
542
|
|
543
|
+
# remove this entry from the sorted set
|
429
544
|
def delete
|
430
545
|
if @value
|
431
546
|
@parent.delete_by_value(@parent.name, @value)
|
@@ -434,11 +549,17 @@ module Sidekiq
|
|
434
549
|
end
|
435
550
|
end
|
436
551
|
|
552
|
+
# Change the scheduled time for this job.
|
553
|
+
#
|
554
|
+
# @param at [Time] the new timestamp for this job
|
437
555
|
def reschedule(at)
|
438
|
-
|
439
|
-
|
556
|
+
Sidekiq.redis do |conn|
|
557
|
+
conn.zincrby(@parent.name, at.to_f - @score, Sidekiq.dump_json(@item))
|
558
|
+
end
|
440
559
|
end
|
441
560
|
|
561
|
+
# Enqueue this job from the scheduled or dead set so it will
|
562
|
+
# be executed at some point in the near future.
|
442
563
|
def add_to_queue
|
443
564
|
remove_job do |message|
|
444
565
|
msg = Sidekiq.load_json(message)
|
@@ -446,16 +567,17 @@ module Sidekiq
|
|
446
567
|
end
|
447
568
|
end
|
448
569
|
|
570
|
+
# enqueue this job from the retry set so it will be executed
|
571
|
+
# at some point in the near future.
|
449
572
|
def retry
|
450
573
|
remove_job do |message|
|
451
574
|
msg = Sidekiq.load_json(message)
|
452
|
-
msg[
|
575
|
+
msg["retry_count"] -= 1 if msg["retry_count"]
|
453
576
|
Sidekiq::Client.push(msg)
|
454
577
|
end
|
455
578
|
end
|
456
579
|
|
457
|
-
|
458
|
-
# Place job in the dead set
|
580
|
+
# Move this job from its current set into the Dead set.
|
459
581
|
def kill
|
460
582
|
remove_job do |message|
|
461
583
|
DeadSet.new.kill(message)
|
@@ -463,74 +585,109 @@ module Sidekiq
|
|
463
585
|
end
|
464
586
|
|
465
587
|
def error?
|
466
|
-
!!item[
|
588
|
+
!!item["error_class"]
|
467
589
|
end
|
468
590
|
|
469
591
|
private
|
470
592
|
|
471
593
|
def remove_job
|
472
594
|
Sidekiq.redis do |conn|
|
473
|
-
results = conn.multi
|
474
|
-
|
475
|
-
|
476
|
-
|
595
|
+
results = conn.multi { |transaction|
|
596
|
+
transaction.zrangebyscore(parent.name, score, score)
|
597
|
+
transaction.zremrangebyscore(parent.name, score, score)
|
598
|
+
}.first
|
477
599
|
|
478
600
|
if results.size == 1
|
479
601
|
yield results.first
|
480
602
|
else
|
481
603
|
# multiple jobs with the same score
|
482
604
|
# find the one with the right JID and push it
|
483
|
-
|
605
|
+
matched, nonmatched = results.partition { |message|
|
484
606
|
if message.index(jid)
|
485
607
|
msg = Sidekiq.load_json(message)
|
486
|
-
msg[
|
608
|
+
msg["jid"] == jid
|
487
609
|
else
|
488
610
|
false
|
489
611
|
end
|
490
|
-
|
612
|
+
}
|
491
613
|
|
492
|
-
msg =
|
614
|
+
msg = matched.first
|
493
615
|
yield msg if msg
|
494
616
|
|
495
617
|
# push the rest back onto the sorted set
|
496
|
-
conn.multi do
|
497
|
-
|
498
|
-
|
618
|
+
conn.multi do |transaction|
|
619
|
+
nonmatched.each do |message|
|
620
|
+
transaction.zadd(parent.name, score.to_f.to_s, message)
|
499
621
|
end
|
500
622
|
end
|
501
623
|
end
|
502
624
|
end
|
503
625
|
end
|
504
|
-
|
505
626
|
end
|
506
627
|
|
628
|
+
# Base class for all sorted sets within Sidekiq.
|
507
629
|
class SortedSet
|
508
630
|
include Enumerable
|
509
631
|
|
632
|
+
# Redis key of the set
|
633
|
+
# @!attribute [r] Name
|
510
634
|
attr_reader :name
|
511
635
|
|
636
|
+
# :nodoc:
|
637
|
+
# @api private
|
512
638
|
def initialize(name)
|
513
639
|
@name = name
|
514
640
|
@_size = size
|
515
641
|
end
|
516
642
|
|
643
|
+
# real-time size of the set, will change
|
517
644
|
def size
|
518
645
|
Sidekiq.redis { |c| c.zcard(name) }
|
519
646
|
end
|
520
647
|
|
648
|
+
# Scan through each element of the sorted set, yielding each to the supplied block.
|
649
|
+
# Please see Redis's <a href="https://redis.io/commands/scan/">SCAN documentation</a> for implementation details.
|
650
|
+
#
|
651
|
+
# @param match [String] a snippet or regexp to filter matches.
|
652
|
+
# @param count [Integer] number of elements to retrieve at a time, default 100
|
653
|
+
# @yieldparam [Sidekiq::SortedEntry] each entry
|
654
|
+
def scan(match, count = 100)
|
655
|
+
return to_enum(:scan, match, count) unless block_given?
|
656
|
+
|
657
|
+
match = "*#{match}*" unless match.include?("*")
|
658
|
+
Sidekiq.redis do |conn|
|
659
|
+
conn.zscan_each(name, match: match, count: count) do |entry, score|
|
660
|
+
yield SortedEntry.new(self, score, entry)
|
661
|
+
end
|
662
|
+
end
|
663
|
+
end
|
664
|
+
|
665
|
+
# @return [Boolean] always true
|
521
666
|
def clear
|
522
667
|
Sidekiq.redis do |conn|
|
523
|
-
conn.
|
668
|
+
conn.unlink(name)
|
524
669
|
end
|
670
|
+
true
|
525
671
|
end
|
526
672
|
alias_method :💣, :clear
|
673
|
+
|
674
|
+
# :nodoc:
|
675
|
+
# @api private
|
676
|
+
def as_json(options = nil)
|
677
|
+
{name: name} # 5336
|
678
|
+
end
|
527
679
|
end
|
528
680
|
|
681
|
+
# Base class for all sorted sets which contain jobs, e.g. scheduled, retry and dead.
|
682
|
+
# Sidekiq Pro and Enterprise add additional sorted sets which do not contain job data,
|
683
|
+
# e.g. Batches.
|
529
684
|
class JobSet < SortedSet
|
530
|
-
|
531
|
-
|
685
|
+
# Add a job with the associated timestamp to this set.
|
686
|
+
# @param timestamp [Time] the score for the job
|
687
|
+
# @param job [Hash] the job data
|
688
|
+
def schedule(timestamp, job)
|
532
689
|
Sidekiq.redis do |conn|
|
533
|
-
conn.zadd(name, timestamp.to_f.to_s, Sidekiq.dump_json(
|
690
|
+
conn.zadd(name, timestamp.to_f.to_s, Sidekiq.dump_json(job))
|
534
691
|
end
|
535
692
|
end
|
536
693
|
|
@@ -540,46 +697,66 @@ module Sidekiq
|
|
540
697
|
page = -1
|
541
698
|
page_size = 50
|
542
699
|
|
543
|
-
|
700
|
+
loop do
|
544
701
|
range_start = page * page_size + offset_size
|
545
|
-
range_end
|
546
|
-
elements = Sidekiq.redis
|
547
|
-
conn.zrange name, range_start, range_end,
|
548
|
-
|
702
|
+
range_end = range_start + page_size - 1
|
703
|
+
elements = Sidekiq.redis { |conn|
|
704
|
+
conn.zrange name, range_start, range_end, withscores: true
|
705
|
+
}
|
549
706
|
break if elements.empty?
|
550
707
|
page -= 1
|
551
|
-
elements.
|
708
|
+
elements.reverse_each do |element, score|
|
552
709
|
yield SortedEntry.new(self, score, element)
|
553
710
|
end
|
554
711
|
offset_size = initial_size - @_size
|
555
712
|
end
|
556
713
|
end
|
557
714
|
|
715
|
+
##
|
716
|
+
# Fetch jobs that match a given time or Range. Job ID is an
|
717
|
+
# optional second argument.
|
718
|
+
#
|
719
|
+
# @param score [Time,Range] a specific timestamp or range
|
720
|
+
# @param jid [String, optional] find a specific JID within the score
|
721
|
+
# @return [Array<SortedEntry>] any results found, can be empty
|
558
722
|
def fetch(score, jid = nil)
|
559
|
-
|
560
|
-
|
561
|
-
|
562
|
-
|
563
|
-
elements.inject([]) do |result, element|
|
564
|
-
entry = SortedEntry.new(self, score, element)
|
565
|
-
if jid
|
566
|
-
result << entry if entry.jid == jid
|
723
|
+
begin_score, end_score =
|
724
|
+
if score.is_a?(Range)
|
725
|
+
[score.first, score.last]
|
567
726
|
else
|
568
|
-
|
727
|
+
[score, score]
|
569
728
|
end
|
570
|
-
|
729
|
+
|
730
|
+
elements = Sidekiq.redis { |conn|
|
731
|
+
conn.zrangebyscore(name, begin_score, end_score, withscores: true)
|
732
|
+
}
|
733
|
+
|
734
|
+
elements.each_with_object([]) do |element, result|
|
735
|
+
data, job_score = element
|
736
|
+
entry = SortedEntry.new(self, job_score, data)
|
737
|
+
result << entry if jid.nil? || entry.jid == jid
|
571
738
|
end
|
572
739
|
end
|
573
740
|
|
574
741
|
##
|
575
742
|
# Find the job with the given JID within this sorted set.
|
743
|
+
# *This is a slow O(n) operation*. Do not use for app logic.
|
576
744
|
#
|
577
|
-
#
|
578
|
-
#
|
745
|
+
# @param jid [String] the job identifier
|
746
|
+
# @return [SortedEntry] the record or nil
|
579
747
|
def find_job(jid)
|
580
|
-
|
748
|
+
Sidekiq.redis do |conn|
|
749
|
+
conn.zscan_each(name, match: "*#{jid}*", count: 100) do |entry, score|
|
750
|
+
job = JSON.parse(entry)
|
751
|
+
matched = job["jid"] == jid
|
752
|
+
return SortedEntry.new(self, score, entry) if matched
|
753
|
+
end
|
754
|
+
end
|
755
|
+
nil
|
581
756
|
end
|
582
757
|
|
758
|
+
# :nodoc:
|
759
|
+
# @api private
|
583
760
|
def delete_by_value(name, value)
|
584
761
|
Sidekiq.redis do |conn|
|
585
762
|
ret = conn.zrem(name, value)
|
@@ -588,17 +765,20 @@ module Sidekiq
|
|
588
765
|
end
|
589
766
|
end
|
590
767
|
|
768
|
+
# :nodoc:
|
769
|
+
# @api private
|
591
770
|
def delete_by_jid(score, jid)
|
592
771
|
Sidekiq.redis do |conn|
|
593
772
|
elements = conn.zrangebyscore(name, score, score)
|
594
773
|
elements.each do |element|
|
595
|
-
|
596
|
-
|
597
|
-
|
598
|
-
|
599
|
-
|
774
|
+
if element.index(jid)
|
775
|
+
message = Sidekiq.load_json(element)
|
776
|
+
if message["jid"] == jid
|
777
|
+
ret = conn.zrem(name, element)
|
778
|
+
@_size -= 1 if ret
|
779
|
+
break ret
|
780
|
+
end
|
600
781
|
end
|
601
|
-
false
|
602
782
|
end
|
603
783
|
end
|
604
784
|
end
|
@@ -607,10 +787,10 @@ module Sidekiq
|
|
607
787
|
end
|
608
788
|
|
609
789
|
##
|
610
|
-
#
|
790
|
+
# The set of scheduled jobs within Sidekiq.
|
611
791
|
# Based on this, you can search/filter for jobs. Here's an
|
612
|
-
# example where I'm selecting
|
613
|
-
# and deleting them from the
|
792
|
+
# example where I'm selecting jobs based on some complex logic
|
793
|
+
# and deleting them from the scheduled set.
|
614
794
|
#
|
615
795
|
# r = Sidekiq::ScheduledSet.new
|
616
796
|
# r.select do |scheduled|
|
@@ -620,12 +800,12 @@ module Sidekiq
|
|
620
800
|
# end.map(&:delete)
|
621
801
|
class ScheduledSet < JobSet
|
622
802
|
def initialize
|
623
|
-
super
|
803
|
+
super "schedule"
|
624
804
|
end
|
625
805
|
end
|
626
806
|
|
627
807
|
##
|
628
|
-
#
|
808
|
+
# The set of retries within Sidekiq.
|
629
809
|
# Based on this, you can search/filter for jobs. Here's an
|
630
810
|
# example where I'm selecting all jobs of a certain type
|
631
811
|
# and deleting them from the retry queue.
|
@@ -638,31 +818,39 @@ module Sidekiq
|
|
638
818
|
# end.map(&:delete)
|
639
819
|
class RetrySet < JobSet
|
640
820
|
def initialize
|
641
|
-
super
|
821
|
+
super "retry"
|
642
822
|
end
|
643
823
|
|
824
|
+
# Enqueues all jobs pending within the retry set.
|
644
825
|
def retry_all
|
645
|
-
while size > 0
|
646
|
-
|
647
|
-
|
826
|
+
each(&:retry) while size > 0
|
827
|
+
end
|
828
|
+
|
829
|
+
# Kills all jobs pending within the retry set.
|
830
|
+
def kill_all
|
831
|
+
each(&:kill) while size > 0
|
648
832
|
end
|
649
833
|
end
|
650
834
|
|
651
835
|
##
|
652
|
-
#
|
836
|
+
# The set of dead jobs within Sidekiq. Dead jobs have failed all of
|
837
|
+
# their retries and are helding in this set pending some sort of manual
|
838
|
+
# fix. They will be removed after 6 months (dead_timeout) if not.
|
653
839
|
#
|
654
840
|
class DeadSet < JobSet
|
655
841
|
def initialize
|
656
|
-
super
|
842
|
+
super "dead"
|
657
843
|
end
|
658
844
|
|
659
|
-
|
845
|
+
# Add the given job to the Dead set.
|
846
|
+
# @param message [String] the job data as JSON
|
847
|
+
def kill(message, opts = {})
|
660
848
|
now = Time.now.to_f
|
661
849
|
Sidekiq.redis do |conn|
|
662
|
-
conn.multi do
|
663
|
-
|
664
|
-
|
665
|
-
|
850
|
+
conn.multi do |transaction|
|
851
|
+
transaction.zadd(name, now.to_s, message)
|
852
|
+
transaction.zremrangebyscore(name, "-inf", now - self.class.timeout)
|
853
|
+
transaction.zremrangebyrank(name, 0, - self.class.max_jobs)
|
666
854
|
end
|
667
855
|
end
|
668
856
|
|
@@ -677,102 +865,130 @@ module Sidekiq
|
|
677
865
|
true
|
678
866
|
end
|
679
867
|
|
868
|
+
# Enqueue all dead jobs
|
680
869
|
def retry_all
|
681
|
-
while size > 0
|
682
|
-
each(&:retry)
|
683
|
-
end
|
870
|
+
each(&:retry) while size > 0
|
684
871
|
end
|
685
872
|
|
873
|
+
# The maximum size of the Dead set. Older entries will be trimmed
|
874
|
+
# to stay within this limit. Default value is 10,000.
|
686
875
|
def self.max_jobs
|
687
|
-
Sidekiq
|
876
|
+
Sidekiq[:dead_max_jobs]
|
688
877
|
end
|
689
878
|
|
879
|
+
# The time limit for entries within the Dead set. Older entries will be thrown away.
|
880
|
+
# Default value is six months.
|
690
881
|
def self.timeout
|
691
|
-
Sidekiq
|
882
|
+
Sidekiq[:dead_timeout_in_seconds]
|
692
883
|
end
|
693
884
|
end
|
694
885
|
|
695
886
|
##
|
696
887
|
# Enumerates the set of Sidekiq processes which are actively working
|
697
|
-
# right now. Each process
|
888
|
+
# right now. Each process sends a heartbeat to Redis every 5 seconds
|
698
889
|
# so this set should be relatively accurate, barring network partitions.
|
699
890
|
#
|
700
|
-
#
|
891
|
+
# @yieldparam [Sidekiq::Process]
|
701
892
|
#
|
702
893
|
class ProcessSet
|
703
894
|
include Enumerable
|
704
895
|
|
705
|
-
|
706
|
-
|
896
|
+
# :nodoc:
|
897
|
+
# @api private
|
898
|
+
def initialize(clean_plz = true)
|
899
|
+
cleanup if clean_plz
|
707
900
|
end
|
708
901
|
|
709
902
|
# Cleans up dead processes recorded in Redis.
|
710
903
|
# Returns the number of processes cleaned.
|
711
|
-
|
904
|
+
# :nodoc:
|
905
|
+
# @api private
|
906
|
+
def cleanup
|
907
|
+
# dont run cleanup more than once per minute
|
908
|
+
return 0 unless Sidekiq.redis { |conn| conn.set("process_cleanup", "1", nx: true, ex: 60) }
|
909
|
+
|
712
910
|
count = 0
|
713
911
|
Sidekiq.redis do |conn|
|
714
|
-
procs = conn.
|
715
|
-
heartbeats = conn.pipelined
|
912
|
+
procs = conn.sscan_each("processes").to_a
|
913
|
+
heartbeats = conn.pipelined { |pipeline|
|
716
914
|
procs.each do |key|
|
717
|
-
|
915
|
+
pipeline.hget(key, "info")
|
718
916
|
end
|
719
|
-
|
917
|
+
}
|
720
918
|
|
721
919
|
# the hash named key has an expiry of 60 seconds.
|
722
920
|
# if it's not found, that means the process has not reported
|
723
921
|
# in to Redis and probably died.
|
724
|
-
to_prune =
|
725
|
-
|
726
|
-
|
727
|
-
|
728
|
-
count = conn.srem('processes', to_prune) unless to_prune.empty?
|
922
|
+
to_prune = procs.select.with_index { |proc, i|
|
923
|
+
heartbeats[i].nil?
|
924
|
+
}
|
925
|
+
count = conn.srem("processes", to_prune) unless to_prune.empty?
|
729
926
|
end
|
730
927
|
count
|
731
928
|
end
|
732
929
|
|
733
930
|
def each
|
734
|
-
|
931
|
+
result = Sidekiq.redis { |conn|
|
932
|
+
procs = conn.sscan_each("processes").to_a.sort
|
735
933
|
|
736
|
-
Sidekiq.redis do |conn|
|
737
934
|
# We're making a tradeoff here between consuming more memory instead of
|
738
935
|
# making more roundtrips to Redis, but if you have hundreds or thousands of workers,
|
739
936
|
# you'll be happier this way
|
740
|
-
|
937
|
+
conn.pipelined do |pipeline|
|
741
938
|
procs.each do |key|
|
742
|
-
|
939
|
+
pipeline.hmget(key, "info", "busy", "beat", "quiet", "rss", "rtt_us")
|
743
940
|
end
|
744
941
|
end
|
942
|
+
}
|
745
943
|
|
746
|
-
|
747
|
-
|
748
|
-
|
749
|
-
|
750
|
-
|
751
|
-
|
752
|
-
|
753
|
-
|
754
|
-
|
944
|
+
result.each do |info, busy, at_s, quiet, rss, rtt|
|
945
|
+
# If a process is stopped between when we query Redis for `procs` and
|
946
|
+
# when we query for `result`, we will have an item in `result` that is
|
947
|
+
# composed of `nil` values.
|
948
|
+
next if info.nil?
|
949
|
+
|
950
|
+
hash = Sidekiq.load_json(info)
|
951
|
+
yield Process.new(hash.merge("busy" => busy.to_i,
|
952
|
+
"beat" => at_s.to_f,
|
953
|
+
"quiet" => quiet,
|
954
|
+
"rss" => rss.to_i,
|
955
|
+
"rtt_us" => rtt.to_i))
|
755
956
|
end
|
756
|
-
|
757
|
-
nil
|
758
957
|
end
|
759
958
|
|
760
959
|
# This method is not guaranteed accurate since it does not prune the set
|
761
960
|
# based on current heartbeat. #each does that and ensures the set only
|
762
961
|
# contains Sidekiq processes which have sent a heartbeat within the last
|
763
962
|
# 60 seconds.
|
963
|
+
# @return [Integer] current number of registered Sidekiq processes
|
764
964
|
def size
|
765
|
-
Sidekiq.redis { |conn| conn.scard(
|
965
|
+
Sidekiq.redis { |conn| conn.scard("processes") }
|
966
|
+
end
|
967
|
+
|
968
|
+
# Total number of threads available to execute jobs.
|
969
|
+
# For Sidekiq Enterprise customers this number (in production) must be
|
970
|
+
# less than or equal to your licensed concurrency.
|
971
|
+
# @return [Integer] the sum of process concurrency
|
972
|
+
def total_concurrency
|
973
|
+
sum { |x| x["concurrency"].to_i }
|
974
|
+
end
|
975
|
+
|
976
|
+
# @return [Integer] total amount of RSS memory consumed by Sidekiq processes
|
977
|
+
def total_rss_in_kb
|
978
|
+
sum { |x| x["rss"].to_i }
|
766
979
|
end
|
980
|
+
alias_method :total_rss, :total_rss_in_kb
|
767
981
|
|
768
982
|
# Returns the identity of the current cluster leader or "" if no leader.
|
769
983
|
# This is a Sidekiq Enterprise feature, will always return "" in Sidekiq
|
770
984
|
# or Sidekiq Pro.
|
985
|
+
# @return [String] Identity of cluster leader
|
986
|
+
# @return [String] empty string if no leader
|
771
987
|
def leader
|
772
988
|
@leader ||= begin
|
773
|
-
x = Sidekiq.redis {|c| c.get("dear-leader") }
|
989
|
+
x = Sidekiq.redis { |c| c.get("dear-leader") }
|
774
990
|
# need a non-falsy value so we can memoize
|
775
|
-
x
|
991
|
+
x ||= ""
|
776
992
|
x
|
777
993
|
end
|
778
994
|
end
|
@@ -794,16 +1010,18 @@ module Sidekiq
|
|
794
1010
|
# 'identity' => <unique string identifying the process>,
|
795
1011
|
# }
|
796
1012
|
class Process
|
1013
|
+
# :nodoc:
|
1014
|
+
# @api private
|
797
1015
|
def initialize(hash)
|
798
1016
|
@attribs = hash
|
799
1017
|
end
|
800
1018
|
|
801
1019
|
def tag
|
802
|
-
self[
|
1020
|
+
self["tag"]
|
803
1021
|
end
|
804
1022
|
|
805
1023
|
def labels
|
806
|
-
Array(self[
|
1024
|
+
Array(self["labels"])
|
807
1025
|
end
|
808
1026
|
|
809
1027
|
def [](key)
|
@@ -811,23 +1029,40 @@ module Sidekiq
|
|
811
1029
|
end
|
812
1030
|
|
813
1031
|
def identity
|
814
|
-
self[
|
1032
|
+
self["identity"]
|
815
1033
|
end
|
816
1034
|
|
1035
|
+
def queues
|
1036
|
+
self["queues"]
|
1037
|
+
end
|
1038
|
+
|
1039
|
+
# Signal this process to stop processing new jobs.
|
1040
|
+
# It will continue to execute jobs it has already fetched.
|
1041
|
+
# This method is *asynchronous* and it can take 5-10
|
1042
|
+
# seconds for the process to quiet.
|
817
1043
|
def quiet!
|
818
|
-
signal(
|
1044
|
+
signal("TSTP")
|
819
1045
|
end
|
820
1046
|
|
1047
|
+
# Signal this process to shutdown.
|
1048
|
+
# It will shutdown within its configured :timeout value, default 25 seconds.
|
1049
|
+
# This method is *asynchronous* and it can take 5-10
|
1050
|
+
# seconds for the process to start shutting down.
|
821
1051
|
def stop!
|
822
|
-
signal(
|
1052
|
+
signal("TERM")
|
823
1053
|
end
|
824
1054
|
|
1055
|
+
# Signal this process to log backtraces for all threads.
|
1056
|
+
# Useful if you have a frozen or deadlocked process which is
|
1057
|
+
# still sending a heartbeat.
|
1058
|
+
# This method is *asynchronous* and it can take 5-10 seconds.
|
825
1059
|
def dump_threads
|
826
|
-
signal(
|
1060
|
+
signal("TTIN")
|
827
1061
|
end
|
828
1062
|
|
1063
|
+
# @return [Boolean] true if this process is quiet or shutting down
|
829
1064
|
def stopping?
|
830
|
-
self[
|
1065
|
+
self["quiet"] == "true"
|
831
1066
|
end
|
832
1067
|
|
833
1068
|
private
|
@@ -835,18 +1070,17 @@ module Sidekiq
|
|
835
1070
|
def signal(sig)
|
836
1071
|
key = "#{identity}-signals"
|
837
1072
|
Sidekiq.redis do |c|
|
838
|
-
c.multi do
|
839
|
-
|
840
|
-
|
1073
|
+
c.multi do |transaction|
|
1074
|
+
transaction.lpush(key, sig)
|
1075
|
+
transaction.expire(key, 60)
|
841
1076
|
end
|
842
1077
|
end
|
843
1078
|
end
|
844
|
-
|
845
1079
|
end
|
846
1080
|
|
847
1081
|
##
|
848
|
-
#
|
849
|
-
#
|
1082
|
+
# The WorkSet stores the work being done by this Sidekiq cluster.
|
1083
|
+
# It tracks the process and thread working on each job.
|
850
1084
|
#
|
851
1085
|
# WARNING WARNING WARNING
|
852
1086
|
#
|
@@ -854,33 +1088,47 @@ module Sidekiq
|
|
854
1088
|
# If you call #size => 5 and then expect #each to be
|
855
1089
|
# called 5 times, you're going to have a bad time.
|
856
1090
|
#
|
857
|
-
#
|
858
|
-
#
|
859
|
-
#
|
1091
|
+
# works = Sidekiq::WorkSet.new
|
1092
|
+
# works.size => 2
|
1093
|
+
# works.each do |process_id, thread_id, work|
|
860
1094
|
# # process_id is a unique identifier per Sidekiq process
|
861
1095
|
# # thread_id is a unique identifier per thread
|
862
1096
|
# # work is a Hash which looks like:
|
863
|
-
# # { 'queue' => name, 'run_at' => timestamp, 'payload' =>
|
1097
|
+
# # { 'queue' => name, 'run_at' => timestamp, 'payload' => job_hash }
|
864
1098
|
# # run_at is an epoch Integer.
|
865
1099
|
# end
|
866
1100
|
#
|
867
|
-
class
|
1101
|
+
class WorkSet
|
868
1102
|
include Enumerable
|
869
1103
|
|
870
|
-
def each
|
1104
|
+
def each(&block)
|
1105
|
+
results = []
|
1106
|
+
procs = nil
|
1107
|
+
all_works = nil
|
1108
|
+
|
871
1109
|
Sidekiq.redis do |conn|
|
872
|
-
procs = conn.
|
873
|
-
|
874
|
-
|
875
|
-
|
876
|
-
|
877
|
-
end
|
878
|
-
next unless valid
|
879
|
-
workers.each_pair do |tid, json|
|
880
|
-
yield key, tid, Sidekiq.load_json(json)
|
1110
|
+
procs = conn.sscan_each("processes").to_a.sort
|
1111
|
+
|
1112
|
+
all_works = conn.pipelined do |pipeline|
|
1113
|
+
procs.each do |key|
|
1114
|
+
pipeline.hgetall("#{key}:work")
|
881
1115
|
end
|
882
1116
|
end
|
883
1117
|
end
|
1118
|
+
|
1119
|
+
procs.zip(all_works).each do |key, workers|
|
1120
|
+
workers.each_pair do |tid, json|
|
1121
|
+
next if json.empty?
|
1122
|
+
|
1123
|
+
hsh = Sidekiq.load_json(json)
|
1124
|
+
p = hsh["payload"]
|
1125
|
+
# avoid breaking API, this is a side effect of the JSON optimization in #4316
|
1126
|
+
hsh["payload"] = Sidekiq.load_json(p) if p.is_a?(String)
|
1127
|
+
results << [key, tid, hsh]
|
1128
|
+
end
|
1129
|
+
end
|
1130
|
+
|
1131
|
+
results.sort_by { |(_, _, hsh)| hsh["run_at"] }.each(&block)
|
884
1132
|
end
|
885
1133
|
|
886
1134
|
# Note that #size is only as accurate as Sidekiq's heartbeat,
|
@@ -891,18 +1139,21 @@ module Sidekiq
|
|
891
1139
|
# which can easily get out of sync with crashy processes.
|
892
1140
|
def size
|
893
1141
|
Sidekiq.redis do |conn|
|
894
|
-
procs = conn.
|
1142
|
+
procs = conn.sscan_each("processes").to_a
|
895
1143
|
if procs.empty?
|
896
1144
|
0
|
897
1145
|
else
|
898
|
-
conn.pipelined
|
1146
|
+
conn.pipelined { |pipeline|
|
899
1147
|
procs.each do |key|
|
900
|
-
|
1148
|
+
pipeline.hget(key, "busy")
|
901
1149
|
end
|
902
|
-
|
1150
|
+
}.sum(&:to_i)
|
903
1151
|
end
|
904
1152
|
end
|
905
1153
|
end
|
906
1154
|
end
|
907
|
-
|
1155
|
+
# Since "worker" is a nebulous term, we've deprecated the use of this class name.
|
1156
|
+
# Is "worker" a process, a type of job, a thread? Undefined!
|
1157
|
+
# WorkSet better describes the data.
|
1158
|
+
Workers = WorkSet
|
908
1159
|
end
|