sidekiq 4.2.10 → 6.5.7
Sign up to get free protection for your applications and to get access to all the features.
Potentially problematic release.
This version of sidekiq might be problematic. Click here for more details.
- checksums.yaml +5 -5
- data/Changes.md +573 -1
- data/LICENSE +3 -3
- data/README.md +25 -34
- data/bin/sidekiq +27 -3
- data/bin/sidekiqload +81 -74
- data/bin/sidekiqmon +8 -0
- data/lib/generators/sidekiq/job_generator.rb +57 -0
- data/lib/generators/sidekiq/templates/{worker.rb.erb → job.rb.erb} +2 -2
- data/lib/generators/sidekiq/templates/job_spec.rb.erb +6 -0
- data/lib/generators/sidekiq/templates/{worker_test.rb.erb → job_test.rb.erb} +1 -1
- data/lib/sidekiq/api.rb +585 -285
- data/lib/sidekiq/cli.rb +256 -233
- data/lib/sidekiq/client.rb +86 -83
- data/lib/sidekiq/component.rb +65 -0
- data/lib/sidekiq/delay.rb +43 -0
- data/lib/sidekiq/extensions/action_mailer.rb +13 -22
- data/lib/sidekiq/extensions/active_record.rb +13 -10
- data/lib/sidekiq/extensions/class_methods.rb +14 -11
- data/lib/sidekiq/extensions/generic_proxy.rb +13 -5
- data/lib/sidekiq/fetch.rb +50 -40
- data/lib/sidekiq/job.rb +13 -0
- data/lib/sidekiq/job_logger.rb +51 -0
- data/lib/sidekiq/job_retry.rb +282 -0
- data/lib/sidekiq/job_util.rb +71 -0
- data/lib/sidekiq/launcher.rb +184 -90
- data/lib/sidekiq/logger.rb +156 -0
- data/lib/sidekiq/manager.rb +43 -45
- data/lib/sidekiq/metrics/deploy.rb +47 -0
- data/lib/sidekiq/metrics/query.rb +153 -0
- data/lib/sidekiq/metrics/shared.rb +94 -0
- data/lib/sidekiq/metrics/tracking.rb +134 -0
- data/lib/sidekiq/middleware/chain.rb +102 -46
- data/lib/sidekiq/middleware/current_attributes.rb +63 -0
- data/lib/sidekiq/middleware/i18n.rb +7 -7
- data/lib/sidekiq/middleware/modules.rb +21 -0
- data/lib/sidekiq/monitor.rb +133 -0
- data/lib/sidekiq/paginator.rb +20 -16
- data/lib/sidekiq/processor.rb +176 -91
- data/lib/sidekiq/rails.rb +41 -96
- data/lib/sidekiq/redis_client_adapter.rb +154 -0
- data/lib/sidekiq/redis_connection.rb +117 -48
- data/lib/sidekiq/ring_buffer.rb +29 -0
- data/lib/sidekiq/scheduled.rb +134 -44
- data/lib/sidekiq/sd_notify.rb +149 -0
- data/lib/sidekiq/systemd.rb +24 -0
- data/lib/sidekiq/testing/inline.rb +6 -5
- data/lib/sidekiq/testing.rb +80 -61
- data/lib/sidekiq/transaction_aware_client.rb +45 -0
- data/lib/sidekiq/version.rb +2 -1
- data/lib/sidekiq/web/action.rb +15 -15
- data/lib/sidekiq/web/application.rb +129 -86
- data/lib/sidekiq/web/csrf_protection.rb +180 -0
- data/lib/sidekiq/web/helpers.rb +170 -83
- data/lib/sidekiq/web/router.rb +23 -19
- data/lib/sidekiq/web.rb +69 -109
- data/lib/sidekiq/worker.rb +290 -41
- data/lib/sidekiq.rb +185 -77
- data/sidekiq.gemspec +23 -27
- data/web/assets/images/apple-touch-icon.png +0 -0
- data/web/assets/javascripts/application.js +112 -61
- data/web/assets/javascripts/chart.min.js +13 -0
- data/web/assets/javascripts/chartjs-plugin-annotation.min.js +7 -0
- data/web/assets/javascripts/dashboard.js +70 -91
- data/web/assets/javascripts/graph.js +16 -0
- data/web/assets/javascripts/metrics.js +262 -0
- data/web/assets/stylesheets/application-dark.css +143 -0
- data/web/assets/stylesheets/application-rtl.css +242 -0
- data/web/assets/stylesheets/application.css +364 -144
- data/web/assets/stylesheets/bootstrap-rtl.min.css +9 -0
- data/web/assets/stylesheets/bootstrap.css +2 -2
- data/web/locales/ar.yml +87 -0
- data/web/locales/de.yml +14 -2
- data/web/locales/el.yml +43 -19
- data/web/locales/en.yml +15 -1
- data/web/locales/es.yml +22 -5
- data/web/locales/fa.yml +1 -0
- data/web/locales/fr.yml +10 -3
- data/web/locales/he.yml +79 -0
- data/web/locales/ja.yml +19 -4
- data/web/locales/lt.yml +83 -0
- data/web/locales/pl.yml +4 -4
- data/web/locales/pt-br.yml +27 -9
- data/web/locales/ru.yml +4 -0
- data/web/locales/ur.yml +80 -0
- data/web/locales/vi.yml +83 -0
- data/web/locales/zh-cn.yml +36 -11
- data/web/locales/zh-tw.yml +32 -7
- data/web/views/_footer.erb +5 -2
- data/web/views/_job_info.erb +3 -2
- data/web/views/_nav.erb +5 -19
- data/web/views/_paging.erb +1 -1
- data/web/views/_poll_link.erb +2 -5
- data/web/views/_summary.erb +7 -7
- data/web/views/busy.erb +62 -24
- data/web/views/dashboard.erb +24 -15
- data/web/views/dead.erb +3 -3
- data/web/views/layout.erb +14 -3
- data/web/views/metrics.erb +69 -0
- data/web/views/metrics_for_job.erb +87 -0
- data/web/views/morgue.erb +9 -6
- data/web/views/queue.erb +26 -12
- data/web/views/queues.erb +12 -2
- data/web/views/retries.erb +14 -7
- data/web/views/retry.erb +3 -3
- data/web/views/scheduled.erb +7 -4
- metadata +66 -206
- data/.github/contributing.md +0 -32
- data/.github/issue_template.md +0 -9
- data/.gitignore +0 -12
- data/.travis.yml +0 -18
- data/3.0-Upgrade.md +0 -70
- data/4.0-Upgrade.md +0 -53
- data/COMM-LICENSE +0 -95
- data/Ent-Changes.md +0 -173
- data/Gemfile +0 -29
- data/Pro-2.0-Upgrade.md +0 -138
- data/Pro-3.0-Upgrade.md +0 -44
- data/Pro-Changes.md +0 -628
- data/Rakefile +0 -12
- data/bin/sidekiqctl +0 -99
- data/code_of_conduct.md +0 -50
- data/lib/generators/sidekiq/templates/worker_spec.rb.erb +0 -6
- data/lib/generators/sidekiq/worker_generator.rb +0 -49
- data/lib/sidekiq/core_ext.rb +0 -119
- data/lib/sidekiq/exception_handler.rb +0 -31
- data/lib/sidekiq/logging.rb +0 -106
- data/lib/sidekiq/middleware/server/active_record.rb +0 -13
- data/lib/sidekiq/middleware/server/logging.rb +0 -31
- data/lib/sidekiq/middleware/server/retry_jobs.rb +0 -205
- data/lib/sidekiq/util.rb +0 -63
data/lib/sidekiq/api.rb
CHANGED
@@ -1,11 +1,36 @@
|
|
1
|
-
# encoding: utf-8
|
2
1
|
# frozen_string_literal: true
|
3
|
-
|
2
|
+
|
3
|
+
require "sidekiq"
|
4
|
+
|
5
|
+
require "zlib"
|
6
|
+
require "set"
|
7
|
+
require "base64"
|
8
|
+
|
9
|
+
if ENV["SIDEKIQ_METRICS_BETA"]
|
10
|
+
require "sidekiq/metrics/deploy"
|
11
|
+
require "sidekiq/metrics/query"
|
12
|
+
end
|
13
|
+
|
14
|
+
#
|
15
|
+
# Sidekiq's Data API provides a Ruby object model on top
|
16
|
+
# of Sidekiq's runtime data in Redis. This API should never
|
17
|
+
# be used within application code for business logic.
|
18
|
+
#
|
19
|
+
# The Sidekiq server process never uses this API: all data
|
20
|
+
# manipulation is done directly for performance reasons to
|
21
|
+
# ensure we are using Redis as efficiently as possible at
|
22
|
+
# every callsite.
|
23
|
+
#
|
4
24
|
|
5
25
|
module Sidekiq
|
26
|
+
# Retrieve runtime statistics from Redis regarding
|
27
|
+
# this Sidekiq cluster.
|
28
|
+
#
|
29
|
+
# stat = Sidekiq::Stats.new
|
30
|
+
# stat.processed
|
6
31
|
class Stats
|
7
32
|
def initialize
|
8
|
-
|
33
|
+
fetch_stats_fast!
|
9
34
|
end
|
10
35
|
|
11
36
|
def processed
|
@@ -48,56 +73,82 @@ module Sidekiq
|
|
48
73
|
Sidekiq::Stats::Queues.new.lengths
|
49
74
|
end
|
50
75
|
|
51
|
-
|
52
|
-
|
53
|
-
|
54
|
-
|
55
|
-
|
56
|
-
|
57
|
-
|
58
|
-
|
59
|
-
|
60
|
-
|
61
|
-
|
62
|
-
|
76
|
+
# O(1) redis calls
|
77
|
+
# @api private
|
78
|
+
def fetch_stats_fast!
|
79
|
+
pipe1_res = Sidekiq.redis { |conn|
|
80
|
+
conn.pipelined do |pipeline|
|
81
|
+
pipeline.get("stat:processed")
|
82
|
+
pipeline.get("stat:failed")
|
83
|
+
pipeline.zcard("schedule")
|
84
|
+
pipeline.zcard("retry")
|
85
|
+
pipeline.zcard("dead")
|
86
|
+
pipeline.scard("processes")
|
87
|
+
pipeline.lrange("queue:default", -1, -1)
|
63
88
|
end
|
64
|
-
|
89
|
+
}
|
65
90
|
|
66
|
-
|
67
|
-
|
68
|
-
|
69
|
-
|
91
|
+
default_queue_latency = if (entry = pipe1_res[6].first)
|
92
|
+
job = begin
|
93
|
+
Sidekiq.load_json(entry)
|
94
|
+
rescue
|
95
|
+
{}
|
70
96
|
end
|
97
|
+
now = Time.now.to_f
|
98
|
+
thence = job["enqueued_at"] || now
|
99
|
+
now - thence
|
100
|
+
else
|
101
|
+
0
|
71
102
|
end
|
72
103
|
|
73
|
-
s = pipe1_res[7].size
|
74
|
-
workers_size = pipe2_res[0...s].map(&:to_i).inject(0, &:+)
|
75
|
-
enqueued = pipe2_res[s..-1].map(&:to_i).inject(0, &:+)
|
76
|
-
|
77
|
-
default_queue_latency = if (entry = pipe1_res[6].first)
|
78
|
-
job = Sidekiq.load_json(entry)
|
79
|
-
now = Time.now.to_f
|
80
|
-
thence = job['enqueued_at'.freeze] || now
|
81
|
-
now - thence
|
82
|
-
else
|
83
|
-
0
|
84
|
-
end
|
85
104
|
@stats = {
|
86
|
-
processed:
|
87
|
-
failed:
|
88
|
-
scheduled_size:
|
89
|
-
retry_size:
|
90
|
-
dead_size:
|
91
|
-
processes_size:
|
92
|
-
|
93
|
-
default_queue_latency: default_queue_latency
|
94
|
-
|
95
|
-
|
105
|
+
processed: pipe1_res[0].to_i,
|
106
|
+
failed: pipe1_res[1].to_i,
|
107
|
+
scheduled_size: pipe1_res[2],
|
108
|
+
retry_size: pipe1_res[3],
|
109
|
+
dead_size: pipe1_res[4],
|
110
|
+
processes_size: pipe1_res[5],
|
111
|
+
|
112
|
+
default_queue_latency: default_queue_latency
|
113
|
+
}
|
114
|
+
end
|
115
|
+
|
116
|
+
# O(number of processes + number of queues) redis calls
|
117
|
+
# @api private
|
118
|
+
def fetch_stats_slow!
|
119
|
+
processes = Sidekiq.redis { |conn|
|
120
|
+
conn.sscan_each("processes").to_a
|
96
121
|
}
|
122
|
+
|
123
|
+
queues = Sidekiq.redis { |conn|
|
124
|
+
conn.sscan_each("queues").to_a
|
125
|
+
}
|
126
|
+
|
127
|
+
pipe2_res = Sidekiq.redis { |conn|
|
128
|
+
conn.pipelined do |pipeline|
|
129
|
+
processes.each { |key| pipeline.hget(key, "busy") }
|
130
|
+
queues.each { |queue| pipeline.llen("queue:#{queue}") }
|
131
|
+
end
|
132
|
+
}
|
133
|
+
|
134
|
+
s = processes.size
|
135
|
+
workers_size = pipe2_res[0...s].sum(&:to_i)
|
136
|
+
enqueued = pipe2_res[s..-1].sum(&:to_i)
|
137
|
+
|
138
|
+
@stats[:workers_size] = workers_size
|
139
|
+
@stats[:enqueued] = enqueued
|
140
|
+
@stats
|
97
141
|
end
|
98
142
|
|
143
|
+
# @api private
|
144
|
+
def fetch_stats!
|
145
|
+
fetch_stats_fast!
|
146
|
+
fetch_stats_slow!
|
147
|
+
end
|
148
|
+
|
149
|
+
# @api private
|
99
150
|
def reset(*stats)
|
100
|
-
all
|
151
|
+
all = %w[failed processed]
|
101
152
|
stats = stats.empty? ? all : all & stats.flatten.compact.map(&:to_s)
|
102
153
|
|
103
154
|
mset_args = []
|
@@ -113,66 +164,62 @@ module Sidekiq
|
|
113
164
|
private
|
114
165
|
|
115
166
|
def stat(s)
|
116
|
-
@stats[s]
|
167
|
+
fetch_stats_slow! if @stats[s].nil?
|
168
|
+
@stats[s] || raise(ArgumentError, "Unknown stat #{s}")
|
117
169
|
end
|
118
170
|
|
119
171
|
class Queues
|
120
172
|
def lengths
|
121
173
|
Sidekiq.redis do |conn|
|
122
|
-
queues = conn.
|
174
|
+
queues = conn.sscan_each("queues").to_a
|
123
175
|
|
124
|
-
lengths = conn.pipelined
|
176
|
+
lengths = conn.pipelined { |pipeline|
|
125
177
|
queues.each do |queue|
|
126
|
-
|
178
|
+
pipeline.llen("queue:#{queue}")
|
127
179
|
end
|
128
|
-
|
180
|
+
}
|
129
181
|
|
130
|
-
|
131
|
-
array_of_arrays
|
132
|
-
memo[queue] = lengths[i]
|
133
|
-
i += 1
|
134
|
-
memo
|
135
|
-
end.sort_by { |_, size| size }
|
136
|
-
|
137
|
-
Hash[array_of_arrays.reverse]
|
182
|
+
array_of_arrays = queues.zip(lengths).sort_by { |_, size| -size }
|
183
|
+
array_of_arrays.to_h
|
138
184
|
end
|
139
185
|
end
|
140
186
|
end
|
141
187
|
|
142
188
|
class History
|
143
189
|
def initialize(days_previous, start_date = nil)
|
190
|
+
# we only store five years of data in Redis
|
191
|
+
raise ArgumentError if days_previous < 1 || days_previous > (5 * 365)
|
144
192
|
@days_previous = days_previous
|
145
193
|
@start_date = start_date || Time.now.utc.to_date
|
146
194
|
end
|
147
195
|
|
148
196
|
def processed
|
149
|
-
date_stat_hash("processed")
|
197
|
+
@processed ||= date_stat_hash("processed")
|
150
198
|
end
|
151
199
|
|
152
200
|
def failed
|
153
|
-
date_stat_hash("failed")
|
201
|
+
@failed ||= date_stat_hash("failed")
|
154
202
|
end
|
155
203
|
|
156
204
|
private
|
157
205
|
|
158
206
|
def date_stat_hash(stat)
|
159
|
-
i = 0
|
160
207
|
stat_hash = {}
|
161
|
-
|
162
|
-
|
163
|
-
|
164
|
-
while i < @days_previous
|
165
|
-
date = @start_date - i
|
166
|
-
datestr = date.strftime("%Y-%m-%d".freeze)
|
167
|
-
keys << "stat:#{stat}:#{datestr}"
|
168
|
-
dates << datestr
|
169
|
-
i += 1
|
170
|
-
end
|
208
|
+
dates = @start_date.downto(@start_date - @days_previous + 1).map { |date|
|
209
|
+
date.strftime("%Y-%m-%d")
|
210
|
+
}
|
171
211
|
|
172
|
-
|
173
|
-
|
174
|
-
|
212
|
+
keys = dates.map { |datestr| "stat:#{stat}:#{datestr}" }
|
213
|
+
|
214
|
+
begin
|
215
|
+
Sidekiq.redis do |conn|
|
216
|
+
conn.mget(keys).each_with_index do |value, idx|
|
217
|
+
stat_hash[dates[idx]] = value ? value.to_i : 0
|
218
|
+
end
|
175
219
|
end
|
220
|
+
rescue RedisConnection.adapter::CommandError
|
221
|
+
# mget will trigger a CROSSSLOT error when run against a Cluster
|
222
|
+
# TODO Someone want to add Cluster support?
|
176
223
|
end
|
177
224
|
|
178
225
|
stat_hash
|
@@ -181,9 +228,10 @@ module Sidekiq
|
|
181
228
|
end
|
182
229
|
|
183
230
|
##
|
184
|
-
#
|
231
|
+
# Represents a queue within Sidekiq.
|
185
232
|
# Allows enumeration of all jobs within the queue
|
186
|
-
# and deletion of jobs.
|
233
|
+
# and deletion of jobs. NB: this queue data is real-time
|
234
|
+
# and is changing within Redis moment by moment.
|
187
235
|
#
|
188
236
|
# queue = Sidekiq::Queue.new("mailer")
|
189
237
|
# queue.each do |job|
|
@@ -191,29 +239,34 @@ module Sidekiq
|
|
191
239
|
# job.args # => [1, 2, 3]
|
192
240
|
# job.delete if job.jid == 'abcdef1234567890'
|
193
241
|
# end
|
194
|
-
#
|
195
242
|
class Queue
|
196
243
|
include Enumerable
|
197
244
|
|
198
245
|
##
|
199
|
-
#
|
246
|
+
# Fetch all known queues within Redis.
|
200
247
|
#
|
248
|
+
# @return [Array<Sidekiq::Queue>]
|
201
249
|
def self.all
|
202
|
-
Sidekiq.redis { |c| c.
|
250
|
+
Sidekiq.redis { |c| c.sscan_each("queues").to_a }.sort.map { |q| Sidekiq::Queue.new(q) }
|
203
251
|
end
|
204
252
|
|
205
253
|
attr_reader :name
|
206
254
|
|
207
|
-
|
208
|
-
|
255
|
+
# @param name [String] the name of the queue
|
256
|
+
def initialize(name = "default")
|
257
|
+
@name = name.to_s
|
209
258
|
@rname = "queue:#{name}"
|
210
259
|
end
|
211
260
|
|
261
|
+
# The current size of the queue within Redis.
|
262
|
+
# This value is real-time and can change between calls.
|
263
|
+
#
|
264
|
+
# @return [Integer] the size
|
212
265
|
def size
|
213
266
|
Sidekiq.redis { |con| con.llen(@rname) }
|
214
267
|
end
|
215
268
|
|
216
|
-
#
|
269
|
+
# @return [Boolean] if the queue is currently paused
|
217
270
|
def paused?
|
218
271
|
false
|
219
272
|
end
|
@@ -222,15 +275,15 @@ module Sidekiq
|
|
222
275
|
# Calculates this queue's latency, the difference in seconds since the oldest
|
223
276
|
# job in the queue was enqueued.
|
224
277
|
#
|
225
|
-
# @return Float
|
278
|
+
# @return [Float] in seconds
|
226
279
|
def latency
|
227
|
-
entry = Sidekiq.redis
|
280
|
+
entry = Sidekiq.redis { |conn|
|
228
281
|
conn.lrange(@rname, -1, -1)
|
229
|
-
|
282
|
+
}.first
|
230
283
|
return 0 unless entry
|
231
284
|
job = Sidekiq.load_json(entry)
|
232
285
|
now = Time.now.to_f
|
233
|
-
thence = job[
|
286
|
+
thence = job["enqueued_at"] || now
|
234
287
|
now - thence
|
235
288
|
end
|
236
289
|
|
@@ -240,16 +293,16 @@ module Sidekiq
|
|
240
293
|
page = 0
|
241
294
|
page_size = 50
|
242
295
|
|
243
|
-
|
296
|
+
loop do
|
244
297
|
range_start = page * page_size - deleted_size
|
245
|
-
range_end
|
246
|
-
entries = Sidekiq.redis
|
298
|
+
range_end = range_start + page_size - 1
|
299
|
+
entries = Sidekiq.redis { |conn|
|
247
300
|
conn.lrange @rname, range_start, range_end
|
248
|
-
|
301
|
+
}
|
249
302
|
break if entries.empty?
|
250
303
|
page += 1
|
251
304
|
entries.each do |entry|
|
252
|
-
yield
|
305
|
+
yield JobRecord.new(entry, @name)
|
253
306
|
end
|
254
307
|
deleted_size = initial_size - size
|
255
308
|
end
|
@@ -258,150 +311,236 @@ module Sidekiq
|
|
258
311
|
##
|
259
312
|
# Find the job with the given JID within this queue.
|
260
313
|
#
|
261
|
-
# This is a slow, inefficient operation. Do not use under
|
262
|
-
# normal conditions.
|
314
|
+
# This is a *slow, inefficient* operation. Do not use under
|
315
|
+
# normal conditions.
|
316
|
+
#
|
317
|
+
# @param jid [String] the job_id to look for
|
318
|
+
# @return [Sidekiq::JobRecord]
|
319
|
+
# @return [nil] if not found
|
263
320
|
def find_job(jid)
|
264
321
|
detect { |j| j.jid == jid }
|
265
322
|
end
|
266
323
|
|
324
|
+
# delete all jobs within this queue
|
325
|
+
# @return [Boolean] true
|
267
326
|
def clear
|
268
327
|
Sidekiq.redis do |conn|
|
269
|
-
conn.multi do
|
270
|
-
|
271
|
-
|
328
|
+
conn.multi do |transaction|
|
329
|
+
transaction.unlink(@rname)
|
330
|
+
transaction.srem("queues", [name])
|
272
331
|
end
|
273
332
|
end
|
333
|
+
true
|
274
334
|
end
|
275
335
|
alias_method :💣, :clear
|
336
|
+
|
337
|
+
# :nodoc:
|
338
|
+
# @api private
|
339
|
+
def as_json(options = nil)
|
340
|
+
{name: name} # 5336
|
341
|
+
end
|
276
342
|
end
|
277
343
|
|
278
344
|
##
|
279
|
-
#
|
280
|
-
# sorted set.
|
345
|
+
# Represents a pending job within a Sidekiq queue.
|
281
346
|
#
|
282
347
|
# The job should be considered immutable but may be
|
283
|
-
# removed from the queue via
|
284
|
-
|
285
|
-
|
348
|
+
# removed from the queue via JobRecord#delete.
|
349
|
+
class JobRecord
|
350
|
+
# the parsed Hash of job data
|
351
|
+
# @!attribute [r] Item
|
286
352
|
attr_reader :item
|
353
|
+
# the underlying String in Redis
|
354
|
+
# @!attribute [r] Value
|
287
355
|
attr_reader :value
|
288
|
-
|
289
|
-
|
356
|
+
# the queue associated with this job
|
357
|
+
# @!attribute [r] Queue
|
358
|
+
attr_reader :queue
|
359
|
+
|
360
|
+
# :nodoc:
|
361
|
+
# @api private
|
362
|
+
def initialize(item, queue_name = nil)
|
363
|
+
@args = nil
|
290
364
|
@value = item
|
291
|
-
@item = item.is_a?(Hash) ? item :
|
292
|
-
@queue = queue_name || @item[
|
365
|
+
@item = item.is_a?(Hash) ? item : parse(item)
|
366
|
+
@queue = queue_name || @item["queue"]
|
293
367
|
end
|
294
368
|
|
369
|
+
# :nodoc:
|
370
|
+
# @api private
|
371
|
+
def parse(item)
|
372
|
+
Sidekiq.load_json(item)
|
373
|
+
rescue JSON::ParserError
|
374
|
+
# If the job payload in Redis is invalid JSON, we'll load
|
375
|
+
# the item as an empty hash and store the invalid JSON as
|
376
|
+
# the job 'args' for display in the Web UI.
|
377
|
+
@invalid = true
|
378
|
+
@args = [item]
|
379
|
+
{}
|
380
|
+
end
|
381
|
+
|
382
|
+
# This is the job class which Sidekiq will execute. If using ActiveJob,
|
383
|
+
# this class will be the ActiveJob adapter class rather than a specific job.
|
295
384
|
def klass
|
296
|
-
|
385
|
+
self["class"]
|
297
386
|
end
|
298
387
|
|
299
388
|
def display_class
|
300
389
|
# Unwrap known wrappers so they show up in a human-friendly manner in the Web UI
|
301
|
-
@klass ||=
|
302
|
-
|
303
|
-
|
304
|
-
|
305
|
-
|
306
|
-
|
307
|
-
|
308
|
-
|
309
|
-
|
310
|
-
|
311
|
-
|
312
|
-
|
313
|
-
|
314
|
-
|
315
|
-
|
316
|
-
|
390
|
+
@klass ||= self["display_class"] || begin
|
391
|
+
case klass
|
392
|
+
when /\ASidekiq::Extensions::Delayed/
|
393
|
+
safe_load(args[0], klass) do |target, method, _|
|
394
|
+
"#{target}.#{method}"
|
395
|
+
end
|
396
|
+
when "ActiveJob::QueueAdapters::SidekiqAdapter::JobWrapper"
|
397
|
+
job_class = @item["wrapped"] || args[0]
|
398
|
+
if job_class == "ActionMailer::DeliveryJob" || job_class == "ActionMailer::MailDeliveryJob"
|
399
|
+
# MailerClass#mailer_method
|
400
|
+
args[0]["arguments"][0..1].join("#")
|
401
|
+
else
|
402
|
+
job_class
|
403
|
+
end
|
404
|
+
else
|
405
|
+
klass
|
406
|
+
end
|
407
|
+
end
|
317
408
|
end
|
318
409
|
|
319
410
|
def display_args
|
320
411
|
# Unwrap known wrappers so they show up in a human-friendly manner in the Web UI
|
321
|
-
@
|
322
|
-
|
323
|
-
|
324
|
-
|
325
|
-
|
326
|
-
|
327
|
-
|
328
|
-
|
329
|
-
|
330
|
-
|
331
|
-
|
332
|
-
|
333
|
-
|
334
|
-
|
335
|
-
|
336
|
-
|
412
|
+
@display_args ||= case klass
|
413
|
+
when /\ASidekiq::Extensions::Delayed/
|
414
|
+
safe_load(args[0], args) do |_, _, arg, kwarg|
|
415
|
+
if !kwarg || kwarg.empty?
|
416
|
+
arg
|
417
|
+
else
|
418
|
+
[arg, kwarg]
|
419
|
+
end
|
420
|
+
end
|
421
|
+
when "ActiveJob::QueueAdapters::SidekiqAdapter::JobWrapper"
|
422
|
+
job_args = self["wrapped"] ? args[0]["arguments"] : []
|
423
|
+
if (self["wrapped"] || args[0]) == "ActionMailer::DeliveryJob"
|
424
|
+
# remove MailerClass, mailer_method and 'deliver_now'
|
425
|
+
job_args.drop(3)
|
426
|
+
elsif (self["wrapped"] || args[0]) == "ActionMailer::MailDeliveryJob"
|
427
|
+
# remove MailerClass, mailer_method and 'deliver_now'
|
428
|
+
job_args.drop(3).first["args"]
|
429
|
+
else
|
430
|
+
job_args
|
431
|
+
end
|
432
|
+
else
|
433
|
+
if self["encrypt"]
|
434
|
+
# no point in showing 150+ bytes of random garbage
|
435
|
+
args[-1] = "[encrypted data]"
|
436
|
+
end
|
437
|
+
args
|
438
|
+
end
|
337
439
|
end
|
338
440
|
|
339
441
|
def args
|
340
|
-
@item[
|
442
|
+
@args || @item["args"]
|
341
443
|
end
|
342
444
|
|
343
445
|
def jid
|
344
|
-
|
446
|
+
self["jid"]
|
345
447
|
end
|
346
448
|
|
347
449
|
def enqueued_at
|
348
|
-
|
450
|
+
self["enqueued_at"] ? Time.at(self["enqueued_at"]).utc : nil
|
349
451
|
end
|
350
452
|
|
351
453
|
def created_at
|
352
|
-
Time.at(
|
454
|
+
Time.at(self["created_at"] || self["enqueued_at"] || 0).utc
|
455
|
+
end
|
456
|
+
|
457
|
+
def tags
|
458
|
+
self["tags"] || []
|
353
459
|
end
|
354
460
|
|
355
|
-
def
|
356
|
-
|
461
|
+
def error_backtrace
|
462
|
+
# Cache nil values
|
463
|
+
if defined?(@error_backtrace)
|
464
|
+
@error_backtrace
|
465
|
+
else
|
466
|
+
value = self["error_backtrace"]
|
467
|
+
@error_backtrace = value && uncompress_backtrace(value)
|
468
|
+
end
|
357
469
|
end
|
358
470
|
|
359
471
|
def latency
|
360
472
|
now = Time.now.to_f
|
361
|
-
now - (@item[
|
473
|
+
now - (@item["enqueued_at"] || @item["created_at"] || now)
|
362
474
|
end
|
363
475
|
|
364
|
-
|
365
|
-
# Remove this job from the queue.
|
476
|
+
# Remove this job from the queue
|
366
477
|
def delete
|
367
|
-
count = Sidekiq.redis
|
478
|
+
count = Sidekiq.redis { |conn|
|
368
479
|
conn.lrem("queue:#{@queue}", 1, @value)
|
369
|
-
|
480
|
+
}
|
370
481
|
count != 0
|
371
482
|
end
|
372
483
|
|
484
|
+
# Access arbitrary attributes within the job hash
|
373
485
|
def [](name)
|
374
|
-
|
486
|
+
# nil will happen if the JSON fails to parse.
|
487
|
+
# We don't guarantee Sidekiq will work with bad job JSON but we should
|
488
|
+
# make a best effort to minimize the damage.
|
489
|
+
@item ? @item[name] : nil
|
375
490
|
end
|
376
491
|
|
377
492
|
private
|
378
493
|
|
379
494
|
def safe_load(content, default)
|
380
|
-
|
381
|
-
|
382
|
-
|
383
|
-
|
384
|
-
|
385
|
-
|
386
|
-
|
495
|
+
yield(*YAML.load(content))
|
496
|
+
rescue => ex
|
497
|
+
# #1761 in dev mode, it's possible to have jobs enqueued which haven't been loaded into
|
498
|
+
# memory yet so the YAML can't be loaded.
|
499
|
+
# TODO is this still necessary? Zeitwerk reloader should handle?
|
500
|
+
Sidekiq.logger.warn "Unable to load YAML: #{ex.message}" unless Sidekiq.options[:environment] == "development"
|
501
|
+
default
|
502
|
+
end
|
503
|
+
|
504
|
+
def uncompress_backtrace(backtrace)
|
505
|
+
if backtrace.is_a?(Array)
|
506
|
+
# Handle old jobs with raw Array backtrace format
|
507
|
+
backtrace
|
508
|
+
else
|
509
|
+
decoded = Base64.decode64(backtrace)
|
510
|
+
uncompressed = Zlib::Inflate.inflate(decoded)
|
511
|
+
begin
|
512
|
+
Sidekiq.load_json(uncompressed)
|
513
|
+
rescue
|
514
|
+
# Handle old jobs with marshalled backtrace format
|
515
|
+
# TODO Remove in 7.x
|
516
|
+
Marshal.load(uncompressed)
|
517
|
+
end
|
387
518
|
end
|
388
519
|
end
|
389
520
|
end
|
390
521
|
|
391
|
-
|
522
|
+
# Represents a job within a Redis sorted set where the score
|
523
|
+
# represents a timestamp associated with the job. This timestamp
|
524
|
+
# could be the scheduled time for it to run (e.g. scheduled set),
|
525
|
+
# or the expiration date after which the entry should be deleted (e.g. dead set).
|
526
|
+
class SortedEntry < JobRecord
|
392
527
|
attr_reader :score
|
393
528
|
attr_reader :parent
|
394
529
|
|
530
|
+
# :nodoc:
|
531
|
+
# @api private
|
395
532
|
def initialize(parent, score, item)
|
396
533
|
super(item)
|
397
|
-
@score = score
|
534
|
+
@score = Float(score)
|
398
535
|
@parent = parent
|
399
536
|
end
|
400
537
|
|
538
|
+
# The timestamp associated with this entry
|
401
539
|
def at
|
402
540
|
Time.at(score).utc
|
403
541
|
end
|
404
542
|
|
543
|
+
# remove this entry from the sorted set
|
405
544
|
def delete
|
406
545
|
if @value
|
407
546
|
@parent.delete_by_value(@parent.name, @value)
|
@@ -410,11 +549,17 @@ module Sidekiq
|
|
410
549
|
end
|
411
550
|
end
|
412
551
|
|
552
|
+
# Change the scheduled time for this job.
|
553
|
+
#
|
554
|
+
# @param at [Time] the new timestamp for this job
|
413
555
|
def reschedule(at)
|
414
|
-
|
415
|
-
|
556
|
+
Sidekiq.redis do |conn|
|
557
|
+
conn.zincrby(@parent.name, at.to_f - @score, Sidekiq.dump_json(@item))
|
558
|
+
end
|
416
559
|
end
|
417
560
|
|
561
|
+
# Enqueue this job from the scheduled or dead set so it will
|
562
|
+
# be executed at some point in the near future.
|
418
563
|
def add_to_queue
|
419
564
|
remove_job do |message|
|
420
565
|
msg = Sidekiq.load_json(message)
|
@@ -422,98 +567,127 @@ module Sidekiq
|
|
422
567
|
end
|
423
568
|
end
|
424
569
|
|
570
|
+
# enqueue this job from the retry set so it will be executed
|
571
|
+
# at some point in the near future.
|
425
572
|
def retry
|
426
573
|
remove_job do |message|
|
427
574
|
msg = Sidekiq.load_json(message)
|
428
|
-
msg[
|
575
|
+
msg["retry_count"] -= 1 if msg["retry_count"]
|
429
576
|
Sidekiq::Client.push(msg)
|
430
577
|
end
|
431
578
|
end
|
432
579
|
|
433
|
-
|
434
|
-
# Place job in the dead set
|
580
|
+
# Move this job from its current set into the Dead set.
|
435
581
|
def kill
|
436
582
|
remove_job do |message|
|
437
|
-
|
438
|
-
Sidekiq.redis do |conn|
|
439
|
-
conn.multi do
|
440
|
-
conn.zadd('dead', now, message)
|
441
|
-
conn.zremrangebyscore('dead', '-inf', now - DeadSet.timeout)
|
442
|
-
conn.zremrangebyrank('dead', 0, - DeadSet.max_jobs)
|
443
|
-
end
|
444
|
-
end
|
583
|
+
DeadSet.new.kill(message)
|
445
584
|
end
|
446
585
|
end
|
447
586
|
|
448
587
|
def error?
|
449
|
-
!!item[
|
588
|
+
!!item["error_class"]
|
450
589
|
end
|
451
590
|
|
452
591
|
private
|
453
592
|
|
454
593
|
def remove_job
|
455
594
|
Sidekiq.redis do |conn|
|
456
|
-
results = conn.multi
|
457
|
-
|
458
|
-
|
459
|
-
|
595
|
+
results = conn.multi { |transaction|
|
596
|
+
transaction.zrangebyscore(parent.name, score, score)
|
597
|
+
transaction.zremrangebyscore(parent.name, score, score)
|
598
|
+
}.first
|
460
599
|
|
461
600
|
if results.size == 1
|
462
601
|
yield results.first
|
463
602
|
else
|
464
603
|
# multiple jobs with the same score
|
465
604
|
# find the one with the right JID and push it
|
466
|
-
|
605
|
+
matched, nonmatched = results.partition { |message|
|
467
606
|
if message.index(jid)
|
468
607
|
msg = Sidekiq.load_json(message)
|
469
|
-
msg[
|
608
|
+
msg["jid"] == jid
|
470
609
|
else
|
471
610
|
false
|
472
611
|
end
|
473
|
-
|
612
|
+
}
|
474
613
|
|
475
|
-
msg =
|
614
|
+
msg = matched.first
|
476
615
|
yield msg if msg
|
477
616
|
|
478
617
|
# push the rest back onto the sorted set
|
479
|
-
conn.multi do
|
480
|
-
|
481
|
-
|
618
|
+
conn.multi do |transaction|
|
619
|
+
nonmatched.each do |message|
|
620
|
+
transaction.zadd(parent.name, score.to_f.to_s, message)
|
482
621
|
end
|
483
622
|
end
|
484
623
|
end
|
485
624
|
end
|
486
625
|
end
|
487
|
-
|
488
626
|
end
|
489
627
|
|
628
|
+
# Base class for all sorted sets within Sidekiq.
|
490
629
|
class SortedSet
|
491
630
|
include Enumerable
|
492
631
|
|
632
|
+
# Redis key of the set
|
633
|
+
# @!attribute [r] Name
|
493
634
|
attr_reader :name
|
494
635
|
|
636
|
+
# :nodoc:
|
637
|
+
# @api private
|
495
638
|
def initialize(name)
|
496
639
|
@name = name
|
497
640
|
@_size = size
|
498
641
|
end
|
499
642
|
|
643
|
+
# real-time size of the set, will change
|
500
644
|
def size
|
501
645
|
Sidekiq.redis { |c| c.zcard(name) }
|
502
646
|
end
|
503
647
|
|
648
|
+
# Scan through each element of the sorted set, yielding each to the supplied block.
|
649
|
+
# Please see Redis's <a href="https://redis.io/commands/scan/">SCAN documentation</a> for implementation details.
|
650
|
+
#
|
651
|
+
# @param match [String] a snippet or regexp to filter matches.
|
652
|
+
# @param count [Integer] number of elements to retrieve at a time, default 100
|
653
|
+
# @yieldparam [Sidekiq::SortedEntry] each entry
|
654
|
+
def scan(match, count = 100)
|
655
|
+
return to_enum(:scan, match, count) unless block_given?
|
656
|
+
|
657
|
+
match = "*#{match}*" unless match.include?("*")
|
658
|
+
Sidekiq.redis do |conn|
|
659
|
+
conn.zscan_each(name, match: match, count: count) do |entry, score|
|
660
|
+
yield SortedEntry.new(self, score, entry)
|
661
|
+
end
|
662
|
+
end
|
663
|
+
end
|
664
|
+
|
665
|
+
# @return [Boolean] always true
|
504
666
|
def clear
|
505
667
|
Sidekiq.redis do |conn|
|
506
|
-
conn.
|
668
|
+
conn.unlink(name)
|
507
669
|
end
|
670
|
+
true
|
508
671
|
end
|
509
672
|
alias_method :💣, :clear
|
673
|
+
|
674
|
+
# :nodoc:
|
675
|
+
# @api private
|
676
|
+
def as_json(options = nil)
|
677
|
+
{name: name} # 5336
|
678
|
+
end
|
510
679
|
end
|
511
680
|
|
681
|
+
# Base class for all sorted sets which contain jobs, e.g. scheduled, retry and dead.
|
682
|
+
# Sidekiq Pro and Enterprise add additional sorted sets which do not contain job data,
|
683
|
+
# e.g. Batches.
|
512
684
|
class JobSet < SortedSet
|
513
|
-
|
514
|
-
|
685
|
+
# Add a job with the associated timestamp to this set.
|
686
|
+
# @param timestamp [Time] the score for the job
|
687
|
+
# @param job [Hash] the job data
|
688
|
+
def schedule(timestamp, job)
|
515
689
|
Sidekiq.redis do |conn|
|
516
|
-
conn.zadd(name, timestamp.to_f.to_s, Sidekiq.dump_json(
|
690
|
+
conn.zadd(name, timestamp.to_f.to_s, Sidekiq.dump_json(job))
|
517
691
|
end
|
518
692
|
end
|
519
693
|
|
@@ -523,46 +697,66 @@ module Sidekiq
|
|
523
697
|
page = -1
|
524
698
|
page_size = 50
|
525
699
|
|
526
|
-
|
700
|
+
loop do
|
527
701
|
range_start = page * page_size + offset_size
|
528
|
-
range_end
|
529
|
-
elements = Sidekiq.redis
|
530
|
-
conn.zrange name, range_start, range_end,
|
531
|
-
|
702
|
+
range_end = range_start + page_size - 1
|
703
|
+
elements = Sidekiq.redis { |conn|
|
704
|
+
conn.zrange name, range_start, range_end, withscores: true
|
705
|
+
}
|
532
706
|
break if elements.empty?
|
533
707
|
page -= 1
|
534
|
-
elements.
|
708
|
+
elements.reverse_each do |element, score|
|
535
709
|
yield SortedEntry.new(self, score, element)
|
536
710
|
end
|
537
711
|
offset_size = initial_size - @_size
|
538
712
|
end
|
539
713
|
end
|
540
714
|
|
715
|
+
##
|
716
|
+
# Fetch jobs that match a given time or Range. Job ID is an
|
717
|
+
# optional second argument.
|
718
|
+
#
|
719
|
+
# @param score [Time,Range] a specific timestamp or range
|
720
|
+
# @param jid [String, optional] find a specific JID within the score
|
721
|
+
# @return [Array<SortedEntry>] any results found, can be empty
|
541
722
|
def fetch(score, jid = nil)
|
542
|
-
|
543
|
-
|
544
|
-
|
545
|
-
|
546
|
-
elements.inject([]) do |result, element|
|
547
|
-
entry = SortedEntry.new(self, score, element)
|
548
|
-
if jid
|
549
|
-
result << entry if entry.jid == jid
|
723
|
+
begin_score, end_score =
|
724
|
+
if score.is_a?(Range)
|
725
|
+
[score.first, score.last]
|
550
726
|
else
|
551
|
-
|
727
|
+
[score, score]
|
552
728
|
end
|
553
|
-
|
729
|
+
|
730
|
+
elements = Sidekiq.redis { |conn|
|
731
|
+
conn.zrangebyscore(name, begin_score, end_score, withscores: true)
|
732
|
+
}
|
733
|
+
|
734
|
+
elements.each_with_object([]) do |element, result|
|
735
|
+
data, job_score = element
|
736
|
+
entry = SortedEntry.new(self, job_score, data)
|
737
|
+
result << entry if jid.nil? || entry.jid == jid
|
554
738
|
end
|
555
739
|
end
|
556
740
|
|
557
741
|
##
|
558
742
|
# Find the job with the given JID within this sorted set.
|
743
|
+
# *This is a slow O(n) operation*. Do not use for app logic.
|
559
744
|
#
|
560
|
-
#
|
561
|
-
#
|
745
|
+
# @param jid [String] the job identifier
|
746
|
+
# @return [SortedEntry] the record or nil
|
562
747
|
def find_job(jid)
|
563
|
-
|
748
|
+
Sidekiq.redis do |conn|
|
749
|
+
conn.zscan_each(name, match: "*#{jid}*", count: 100) do |entry, score|
|
750
|
+
job = JSON.parse(entry)
|
751
|
+
matched = job["jid"] == jid
|
752
|
+
return SortedEntry.new(self, score, entry) if matched
|
753
|
+
end
|
754
|
+
end
|
755
|
+
nil
|
564
756
|
end
|
565
757
|
|
758
|
+
# :nodoc:
|
759
|
+
# @api private
|
566
760
|
def delete_by_value(name, value)
|
567
761
|
Sidekiq.redis do |conn|
|
568
762
|
ret = conn.zrem(name, value)
|
@@ -571,17 +765,20 @@ module Sidekiq
|
|
571
765
|
end
|
572
766
|
end
|
573
767
|
|
768
|
+
# :nodoc:
|
769
|
+
# @api private
|
574
770
|
def delete_by_jid(score, jid)
|
575
771
|
Sidekiq.redis do |conn|
|
576
772
|
elements = conn.zrangebyscore(name, score, score)
|
577
773
|
elements.each do |element|
|
578
|
-
|
579
|
-
|
580
|
-
|
581
|
-
|
582
|
-
|
774
|
+
if element.index(jid)
|
775
|
+
message = Sidekiq.load_json(element)
|
776
|
+
if message["jid"] == jid
|
777
|
+
ret = conn.zrem(name, element)
|
778
|
+
@_size -= 1 if ret
|
779
|
+
break ret
|
780
|
+
end
|
583
781
|
end
|
584
|
-
false
|
585
782
|
end
|
586
783
|
end
|
587
784
|
end
|
@@ -590,10 +787,10 @@ module Sidekiq
|
|
590
787
|
end
|
591
788
|
|
592
789
|
##
|
593
|
-
#
|
790
|
+
# The set of scheduled jobs within Sidekiq.
|
594
791
|
# Based on this, you can search/filter for jobs. Here's an
|
595
|
-
# example where I'm selecting
|
596
|
-
# and deleting them from the
|
792
|
+
# example where I'm selecting jobs based on some complex logic
|
793
|
+
# and deleting them from the scheduled set.
|
597
794
|
#
|
598
795
|
# r = Sidekiq::ScheduledSet.new
|
599
796
|
# r.select do |scheduled|
|
@@ -603,12 +800,12 @@ module Sidekiq
|
|
603
800
|
# end.map(&:delete)
|
604
801
|
class ScheduledSet < JobSet
|
605
802
|
def initialize
|
606
|
-
super
|
803
|
+
super "schedule"
|
607
804
|
end
|
608
805
|
end
|
609
806
|
|
610
807
|
##
|
611
|
-
#
|
808
|
+
# The set of retries within Sidekiq.
|
612
809
|
# Based on this, you can search/filter for jobs. Here's an
|
613
810
|
# example where I'm selecting all jobs of a certain type
|
614
811
|
# and deleting them from the retry queue.
|
@@ -621,105 +818,179 @@ module Sidekiq
|
|
621
818
|
# end.map(&:delete)
|
622
819
|
class RetrySet < JobSet
|
623
820
|
def initialize
|
624
|
-
super
|
821
|
+
super "retry"
|
625
822
|
end
|
626
823
|
|
824
|
+
# Enqueues all jobs pending within the retry set.
|
627
825
|
def retry_all
|
628
|
-
while size > 0
|
629
|
-
|
630
|
-
|
826
|
+
each(&:retry) while size > 0
|
827
|
+
end
|
828
|
+
|
829
|
+
# Kills all jobs pending within the retry set.
|
830
|
+
def kill_all
|
831
|
+
each(&:kill) while size > 0
|
631
832
|
end
|
632
833
|
end
|
633
834
|
|
634
835
|
##
|
635
|
-
#
|
836
|
+
# The set of dead jobs within Sidekiq. Dead jobs have failed all of
|
837
|
+
# their retries and are helding in this set pending some sort of manual
|
838
|
+
# fix. They will be removed after 6 months (dead_timeout) if not.
|
636
839
|
#
|
637
840
|
class DeadSet < JobSet
|
638
841
|
def initialize
|
639
|
-
super
|
842
|
+
super "dead"
|
640
843
|
end
|
641
844
|
|
642
|
-
|
643
|
-
|
644
|
-
|
845
|
+
# Add the given job to the Dead set.
|
846
|
+
# @param message [String] the job data as JSON
|
847
|
+
def kill(message, opts = {})
|
848
|
+
now = Time.now.to_f
|
849
|
+
Sidekiq.redis do |conn|
|
850
|
+
conn.multi do |transaction|
|
851
|
+
transaction.zadd(name, now.to_s, message)
|
852
|
+
transaction.zremrangebyscore(name, "-inf", now - self.class.timeout)
|
853
|
+
transaction.zremrangebyrank(name, 0, - self.class.max_jobs)
|
854
|
+
end
|
645
855
|
end
|
856
|
+
|
857
|
+
if opts[:notify_failure] != false
|
858
|
+
job = Sidekiq.load_json(message)
|
859
|
+
r = RuntimeError.new("Job killed by API")
|
860
|
+
r.set_backtrace(caller)
|
861
|
+
Sidekiq.death_handlers.each do |handle|
|
862
|
+
handle.call(job, r)
|
863
|
+
end
|
864
|
+
end
|
865
|
+
true
|
646
866
|
end
|
647
867
|
|
868
|
+
# Enqueue all dead jobs
|
869
|
+
def retry_all
|
870
|
+
each(&:retry) while size > 0
|
871
|
+
end
|
872
|
+
|
873
|
+
# The maximum size of the Dead set. Older entries will be trimmed
|
874
|
+
# to stay within this limit. Default value is 10,000.
|
648
875
|
def self.max_jobs
|
649
|
-
Sidekiq
|
876
|
+
Sidekiq[:dead_max_jobs]
|
650
877
|
end
|
651
878
|
|
879
|
+
# The time limit for entries within the Dead set. Older entries will be thrown away.
|
880
|
+
# Default value is six months.
|
652
881
|
def self.timeout
|
653
|
-
Sidekiq
|
882
|
+
Sidekiq[:dead_timeout_in_seconds]
|
654
883
|
end
|
655
884
|
end
|
656
885
|
|
657
886
|
##
|
658
887
|
# Enumerates the set of Sidekiq processes which are actively working
|
659
|
-
# right now. Each process
|
888
|
+
# right now. Each process sends a heartbeat to Redis every 5 seconds
|
660
889
|
# so this set should be relatively accurate, barring network partitions.
|
661
890
|
#
|
662
|
-
#
|
891
|
+
# @yieldparam [Sidekiq::Process]
|
663
892
|
#
|
664
893
|
class ProcessSet
|
665
894
|
include Enumerable
|
666
895
|
|
667
|
-
|
668
|
-
|
896
|
+
# :nodoc:
|
897
|
+
# @api private
|
898
|
+
def initialize(clean_plz = true)
|
899
|
+
cleanup if clean_plz
|
669
900
|
end
|
670
901
|
|
671
902
|
# Cleans up dead processes recorded in Redis.
|
672
903
|
# Returns the number of processes cleaned.
|
673
|
-
|
904
|
+
# :nodoc:
|
905
|
+
# @api private
|
906
|
+
def cleanup
|
907
|
+
# dont run cleanup more than once per minute
|
908
|
+
return 0 unless Sidekiq.redis { |conn| conn.set("process_cleanup", "1", nx: true, ex: 60) }
|
909
|
+
|
674
910
|
count = 0
|
675
911
|
Sidekiq.redis do |conn|
|
676
|
-
procs = conn.
|
677
|
-
heartbeats = conn.pipelined
|
912
|
+
procs = conn.sscan_each("processes").to_a
|
913
|
+
heartbeats = conn.pipelined { |pipeline|
|
678
914
|
procs.each do |key|
|
679
|
-
|
915
|
+
pipeline.hget(key, "info")
|
680
916
|
end
|
681
|
-
|
917
|
+
}
|
682
918
|
|
683
919
|
# the hash named key has an expiry of 60 seconds.
|
684
920
|
# if it's not found, that means the process has not reported
|
685
921
|
# in to Redis and probably died.
|
686
|
-
to_prune =
|
687
|
-
|
688
|
-
|
689
|
-
|
690
|
-
count = conn.srem('processes', to_prune) unless to_prune.empty?
|
922
|
+
to_prune = procs.select.with_index { |proc, i|
|
923
|
+
heartbeats[i].nil?
|
924
|
+
}
|
925
|
+
count = conn.srem("processes", to_prune) unless to_prune.empty?
|
691
926
|
end
|
692
927
|
count
|
693
928
|
end
|
694
929
|
|
695
930
|
def each
|
696
|
-
|
931
|
+
result = Sidekiq.redis { |conn|
|
932
|
+
procs = conn.sscan_each("processes").to_a.sort
|
697
933
|
|
698
|
-
Sidekiq.redis do |conn|
|
699
934
|
# We're making a tradeoff here between consuming more memory instead of
|
700
935
|
# making more roundtrips to Redis, but if you have hundreds or thousands of workers,
|
701
936
|
# you'll be happier this way
|
702
|
-
|
937
|
+
conn.pipelined do |pipeline|
|
703
938
|
procs.each do |key|
|
704
|
-
|
939
|
+
pipeline.hmget(key, "info", "busy", "beat", "quiet", "rss", "rtt_us")
|
705
940
|
end
|
706
941
|
end
|
942
|
+
}
|
707
943
|
|
708
|
-
|
709
|
-
|
710
|
-
|
711
|
-
|
944
|
+
result.each do |info, busy, at_s, quiet, rss, rtt|
|
945
|
+
# If a process is stopped between when we query Redis for `procs` and
|
946
|
+
# when we query for `result`, we will have an item in `result` that is
|
947
|
+
# composed of `nil` values.
|
948
|
+
next if info.nil?
|
949
|
+
|
950
|
+
hash = Sidekiq.load_json(info)
|
951
|
+
yield Process.new(hash.merge("busy" => busy.to_i,
|
952
|
+
"beat" => at_s.to_f,
|
953
|
+
"quiet" => quiet,
|
954
|
+
"rss" => rss.to_i,
|
955
|
+
"rtt_us" => rtt.to_i))
|
712
956
|
end
|
713
|
-
|
714
|
-
nil
|
715
957
|
end
|
716
958
|
|
717
959
|
# This method is not guaranteed accurate since it does not prune the set
|
718
960
|
# based on current heartbeat. #each does that and ensures the set only
|
719
961
|
# contains Sidekiq processes which have sent a heartbeat within the last
|
720
962
|
# 60 seconds.
|
963
|
+
# @return [Integer] current number of registered Sidekiq processes
|
721
964
|
def size
|
722
|
-
Sidekiq.redis { |conn| conn.scard(
|
965
|
+
Sidekiq.redis { |conn| conn.scard("processes") }
|
966
|
+
end
|
967
|
+
|
968
|
+
# Total number of threads available to execute jobs.
|
969
|
+
# For Sidekiq Enterprise customers this number (in production) must be
|
970
|
+
# less than or equal to your licensed concurrency.
|
971
|
+
# @return [Integer] the sum of process concurrency
|
972
|
+
def total_concurrency
|
973
|
+
sum { |x| x["concurrency"].to_i }
|
974
|
+
end
|
975
|
+
|
976
|
+
# @return [Integer] total amount of RSS memory consumed by Sidekiq processes
|
977
|
+
def total_rss_in_kb
|
978
|
+
sum { |x| x["rss"].to_i }
|
979
|
+
end
|
980
|
+
alias_method :total_rss, :total_rss_in_kb
|
981
|
+
|
982
|
+
# Returns the identity of the current cluster leader or "" if no leader.
|
983
|
+
# This is a Sidekiq Enterprise feature, will always return "" in Sidekiq
|
984
|
+
# or Sidekiq Pro.
|
985
|
+
# @return [String] Identity of cluster leader
|
986
|
+
# @return [String] empty string if no leader
|
987
|
+
def leader
|
988
|
+
@leader ||= begin
|
989
|
+
x = Sidekiq.redis { |c| c.get("dear-leader") }
|
990
|
+
# need a non-falsy value so we can memoize
|
991
|
+
x ||= ""
|
992
|
+
x
|
993
|
+
end
|
723
994
|
end
|
724
995
|
end
|
725
996
|
|
@@ -739,36 +1010,59 @@ module Sidekiq
|
|
739
1010
|
# 'identity' => <unique string identifying the process>,
|
740
1011
|
# }
|
741
1012
|
class Process
|
1013
|
+
# :nodoc:
|
1014
|
+
# @api private
|
742
1015
|
def initialize(hash)
|
743
1016
|
@attribs = hash
|
744
1017
|
end
|
745
1018
|
|
746
1019
|
def tag
|
747
|
-
self[
|
1020
|
+
self["tag"]
|
748
1021
|
end
|
749
1022
|
|
750
1023
|
def labels
|
751
|
-
Array(self[
|
1024
|
+
Array(self["labels"])
|
752
1025
|
end
|
753
1026
|
|
754
1027
|
def [](key)
|
755
1028
|
@attribs[key]
|
756
1029
|
end
|
757
1030
|
|
1031
|
+
def identity
|
1032
|
+
self["identity"]
|
1033
|
+
end
|
1034
|
+
|
1035
|
+
def queues
|
1036
|
+
self["queues"]
|
1037
|
+
end
|
1038
|
+
|
1039
|
+
# Signal this process to stop processing new jobs.
|
1040
|
+
# It will continue to execute jobs it has already fetched.
|
1041
|
+
# This method is *asynchronous* and it can take 5-10
|
1042
|
+
# seconds for the process to quiet.
|
758
1043
|
def quiet!
|
759
|
-
signal(
|
1044
|
+
signal("TSTP")
|
760
1045
|
end
|
761
1046
|
|
1047
|
+
# Signal this process to shutdown.
|
1048
|
+
# It will shutdown within its configured :timeout value, default 25 seconds.
|
1049
|
+
# This method is *asynchronous* and it can take 5-10
|
1050
|
+
# seconds for the process to start shutting down.
|
762
1051
|
def stop!
|
763
|
-
signal(
|
1052
|
+
signal("TERM")
|
764
1053
|
end
|
765
1054
|
|
1055
|
+
# Signal this process to log backtraces for all threads.
|
1056
|
+
# Useful if you have a frozen or deadlocked process which is
|
1057
|
+
# still sending a heartbeat.
|
1058
|
+
# This method is *asynchronous* and it can take 5-10 seconds.
|
766
1059
|
def dump_threads
|
767
|
-
signal(
|
1060
|
+
signal("TTIN")
|
768
1061
|
end
|
769
1062
|
|
1063
|
+
# @return [Boolean] true if this process is quiet or shutting down
|
770
1064
|
def stopping?
|
771
|
-
self[
|
1065
|
+
self["quiet"] == "true"
|
772
1066
|
end
|
773
1067
|
|
774
1068
|
private
|
@@ -776,21 +1070,17 @@ module Sidekiq
|
|
776
1070
|
def signal(sig)
|
777
1071
|
key = "#{identity}-signals"
|
778
1072
|
Sidekiq.redis do |c|
|
779
|
-
c.multi do
|
780
|
-
|
781
|
-
|
1073
|
+
c.multi do |transaction|
|
1074
|
+
transaction.lpush(key, sig)
|
1075
|
+
transaction.expire(key, 60)
|
782
1076
|
end
|
783
1077
|
end
|
784
1078
|
end
|
785
|
-
|
786
|
-
def identity
|
787
|
-
self['identity']
|
788
|
-
end
|
789
1079
|
end
|
790
1080
|
|
791
1081
|
##
|
792
|
-
#
|
793
|
-
#
|
1082
|
+
# The WorkSet stores the work being done by this Sidekiq cluster.
|
1083
|
+
# It tracks the process and thread working on each job.
|
794
1084
|
#
|
795
1085
|
# WARNING WARNING WARNING
|
796
1086
|
#
|
@@ -798,33 +1088,40 @@ module Sidekiq
|
|
798
1088
|
# If you call #size => 5 and then expect #each to be
|
799
1089
|
# called 5 times, you're going to have a bad time.
|
800
1090
|
#
|
801
|
-
#
|
802
|
-
#
|
803
|
-
#
|
1091
|
+
# works = Sidekiq::WorkSet.new
|
1092
|
+
# works.size => 2
|
1093
|
+
# works.each do |process_id, thread_id, work|
|
804
1094
|
# # process_id is a unique identifier per Sidekiq process
|
805
1095
|
# # thread_id is a unique identifier per thread
|
806
1096
|
# # work is a Hash which looks like:
|
807
|
-
# # { 'queue' => name, 'run_at' => timestamp, 'payload' =>
|
1097
|
+
# # { 'queue' => name, 'run_at' => timestamp, 'payload' => job_hash }
|
808
1098
|
# # run_at is an epoch Integer.
|
809
1099
|
# end
|
810
1100
|
#
|
811
|
-
class
|
1101
|
+
class WorkSet
|
812
1102
|
include Enumerable
|
813
1103
|
|
814
|
-
def each
|
1104
|
+
def each(&block)
|
1105
|
+
results = []
|
815
1106
|
Sidekiq.redis do |conn|
|
816
|
-
procs = conn.
|
1107
|
+
procs = conn.sscan_each("processes").to_a
|
817
1108
|
procs.sort.each do |key|
|
818
|
-
valid, workers = conn.pipelined
|
819
|
-
|
820
|
-
|
821
|
-
|
1109
|
+
valid, workers = conn.pipelined { |pipeline|
|
1110
|
+
pipeline.exists?(key)
|
1111
|
+
pipeline.hgetall("#{key}:work")
|
1112
|
+
}
|
822
1113
|
next unless valid
|
823
1114
|
workers.each_pair do |tid, json|
|
824
|
-
|
1115
|
+
hsh = Sidekiq.load_json(json)
|
1116
|
+
p = hsh["payload"]
|
1117
|
+
# avoid breaking API, this is a side effect of the JSON optimization in #4316
|
1118
|
+
hsh["payload"] = Sidekiq.load_json(p) if p.is_a?(String)
|
1119
|
+
results << [key, tid, hsh]
|
825
1120
|
end
|
826
1121
|
end
|
827
1122
|
end
|
1123
|
+
|
1124
|
+
results.sort_by { |(_, _, hsh)| hsh["run_at"] }.each(&block)
|
828
1125
|
end
|
829
1126
|
|
830
1127
|
# Note that #size is only as accurate as Sidekiq's heartbeat,
|
@@ -835,18 +1132,21 @@ module Sidekiq
|
|
835
1132
|
# which can easily get out of sync with crashy processes.
|
836
1133
|
def size
|
837
1134
|
Sidekiq.redis do |conn|
|
838
|
-
procs = conn.
|
1135
|
+
procs = conn.sscan_each("processes").to_a
|
839
1136
|
if procs.empty?
|
840
1137
|
0
|
841
1138
|
else
|
842
|
-
conn.pipelined
|
1139
|
+
conn.pipelined { |pipeline|
|
843
1140
|
procs.each do |key|
|
844
|
-
|
1141
|
+
pipeline.hget(key, "busy")
|
845
1142
|
end
|
846
|
-
|
1143
|
+
}.sum(&:to_i)
|
847
1144
|
end
|
848
1145
|
end
|
849
1146
|
end
|
850
1147
|
end
|
851
|
-
|
1148
|
+
# Since "worker" is a nebulous term, we've deprecated the use of this class name.
|
1149
|
+
# Is "worker" a process, a type of job, a thread? Undefined!
|
1150
|
+
# WorkSet better describes the data.
|
1151
|
+
Workers = WorkSet
|
852
1152
|
end
|