sidekiq 4.2.4 → 6.4.0
Sign up to get free protection for your applications and to get access to all the features.
Potentially problematic release.
This version of sidekiq might be problematic. Click here for more details.
- checksums.yaml +5 -5
- data/Changes.md +523 -0
- data/LICENSE +3 -3
- data/README.md +23 -36
- data/bin/sidekiq +26 -2
- data/bin/sidekiqload +28 -38
- data/bin/sidekiqmon +8 -0
- data/lib/generators/sidekiq/job_generator.rb +57 -0
- data/lib/generators/sidekiq/templates/{worker.rb.erb → job.rb.erb} +2 -2
- data/lib/generators/sidekiq/templates/job_spec.rb.erb +6 -0
- data/lib/generators/sidekiq/templates/job_test.rb.erb +8 -0
- data/lib/sidekiq/api.rb +403 -243
- data/lib/sidekiq/cli.rb +230 -211
- data/lib/sidekiq/client.rb +53 -64
- data/lib/sidekiq/delay.rb +43 -0
- data/lib/sidekiq/exception_handler.rb +12 -16
- data/lib/sidekiq/extensions/action_mailer.rb +15 -24
- data/lib/sidekiq/extensions/active_record.rb +15 -12
- data/lib/sidekiq/extensions/class_methods.rb +16 -13
- data/lib/sidekiq/extensions/generic_proxy.rb +14 -6
- data/lib/sidekiq/fetch.rb +39 -31
- data/lib/sidekiq/job.rb +13 -0
- data/lib/sidekiq/job_logger.rb +63 -0
- data/lib/sidekiq/job_retry.rb +261 -0
- data/lib/sidekiq/job_util.rb +65 -0
- data/lib/sidekiq/launcher.rb +170 -71
- data/lib/sidekiq/logger.rb +166 -0
- data/lib/sidekiq/manager.rb +21 -26
- data/lib/sidekiq/middleware/chain.rb +20 -8
- data/lib/sidekiq/middleware/current_attributes.rb +57 -0
- data/lib/sidekiq/middleware/i18n.rb +5 -7
- data/lib/sidekiq/monitor.rb +133 -0
- data/lib/sidekiq/paginator.rb +18 -14
- data/lib/sidekiq/processor.rb +161 -70
- data/lib/sidekiq/rails.rb +41 -73
- data/lib/sidekiq/redis_connection.rb +65 -20
- data/lib/sidekiq/scheduled.rb +95 -34
- data/lib/sidekiq/sd_notify.rb +149 -0
- data/lib/sidekiq/systemd.rb +24 -0
- data/lib/sidekiq/testing/inline.rb +2 -1
- data/lib/sidekiq/testing.rb +52 -26
- data/lib/sidekiq/util.rb +60 -14
- data/lib/sidekiq/version.rb +2 -1
- data/lib/sidekiq/web/action.rb +15 -15
- data/lib/sidekiq/web/application.rb +115 -89
- data/lib/sidekiq/web/csrf_protection.rb +180 -0
- data/lib/sidekiq/web/helpers.rb +151 -83
- data/lib/sidekiq/web/router.rb +27 -19
- data/lib/sidekiq/web.rb +65 -109
- data/lib/sidekiq/worker.rb +284 -41
- data/lib/sidekiq.rb +93 -60
- data/sidekiq.gemspec +24 -22
- data/web/assets/images/apple-touch-icon.png +0 -0
- data/web/assets/javascripts/application.js +83 -64
- data/web/assets/javascripts/dashboard.js +81 -85
- data/web/assets/stylesheets/application-dark.css +143 -0
- data/web/assets/stylesheets/application-rtl.css +242 -0
- data/web/assets/stylesheets/application.css +319 -143
- data/web/assets/stylesheets/bootstrap-rtl.min.css +9 -0
- data/web/assets/stylesheets/bootstrap.css +2 -2
- data/web/locales/ar.yml +87 -0
- data/web/locales/de.yml +14 -2
- data/web/locales/en.yml +8 -1
- data/web/locales/es.yml +22 -5
- data/web/locales/fa.yml +80 -0
- data/web/locales/fr.yml +10 -3
- data/web/locales/he.yml +79 -0
- data/web/locales/ja.yml +12 -4
- data/web/locales/lt.yml +83 -0
- data/web/locales/pl.yml +4 -4
- data/web/locales/ru.yml +4 -0
- data/web/locales/ur.yml +80 -0
- data/web/locales/vi.yml +83 -0
- data/web/views/_footer.erb +5 -2
- data/web/views/_job_info.erb +4 -3
- data/web/views/_nav.erb +4 -18
- data/web/views/_paging.erb +1 -1
- data/web/views/_poll_link.erb +2 -5
- data/web/views/_summary.erb +7 -7
- data/web/views/busy.erb +60 -22
- data/web/views/dashboard.erb +23 -15
- data/web/views/dead.erb +3 -3
- data/web/views/layout.erb +14 -3
- data/web/views/morgue.erb +19 -12
- data/web/views/queue.erb +24 -14
- data/web/views/queues.erb +14 -4
- data/web/views/retries.erb +22 -13
- data/web/views/retry.erb +4 -4
- data/web/views/scheduled.erb +7 -4
- metadata +49 -198
- data/.github/contributing.md +0 -32
- data/.github/issue_template.md +0 -4
- data/.gitignore +0 -12
- data/.travis.yml +0 -12
- data/3.0-Upgrade.md +0 -70
- data/4.0-Upgrade.md +0 -53
- data/COMM-LICENSE +0 -95
- data/Ent-Changes.md +0 -146
- data/Gemfile +0 -29
- data/Pro-2.0-Upgrade.md +0 -138
- data/Pro-3.0-Upgrade.md +0 -44
- data/Pro-Changes.md +0 -585
- data/Rakefile +0 -9
- data/bin/sidekiqctl +0 -99
- data/code_of_conduct.md +0 -50
- data/lib/generators/sidekiq/templates/worker_spec.rb.erb +0 -6
- data/lib/generators/sidekiq/templates/worker_test.rb.erb +0 -8
- data/lib/generators/sidekiq/worker_generator.rb +0 -49
- data/lib/sidekiq/core_ext.rb +0 -106
- data/lib/sidekiq/logging.rb +0 -106
- data/lib/sidekiq/middleware/server/active_record.rb +0 -13
- data/lib/sidekiq/middleware/server/logging.rb +0 -40
- data/lib/sidekiq/middleware/server/retry_jobs.rb +0 -205
- data/test/config.yml +0 -9
- data/test/env_based_config.yml +0 -11
- data/test/fake_env.rb +0 -1
- data/test/fixtures/en.yml +0 -2
- data/test/helper.rb +0 -75
- data/test/test_actors.rb +0 -138
- data/test/test_api.rb +0 -528
- data/test/test_cli.rb +0 -418
- data/test/test_client.rb +0 -266
- data/test/test_exception_handler.rb +0 -56
- data/test/test_extensions.rb +0 -127
- data/test/test_fetch.rb +0 -50
- data/test/test_launcher.rb +0 -95
- data/test/test_logging.rb +0 -35
- data/test/test_manager.rb +0 -50
- data/test/test_middleware.rb +0 -158
- data/test/test_processor.rb +0 -235
- data/test/test_rails.rb +0 -22
- data/test/test_redis_connection.rb +0 -132
- data/test/test_retry.rb +0 -326
- data/test/test_retry_exhausted.rb +0 -149
- data/test/test_scheduled.rb +0 -115
- data/test/test_scheduling.rb +0 -58
- data/test/test_sidekiq.rb +0 -107
- data/test/test_testing.rb +0 -143
- data/test/test_testing_fake.rb +0 -357
- data/test/test_testing_inline.rb +0 -94
- data/test/test_util.rb +0 -13
- data/test/test_web.rb +0 -726
- data/test/test_web_helpers.rb +0 -54
data/lib/sidekiq/api.rb
CHANGED
@@ -1,11 +1,14 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
|
-
|
3
|
-
require
|
2
|
+
|
3
|
+
require "sidekiq"
|
4
|
+
|
5
|
+
require "zlib"
|
6
|
+
require "base64"
|
4
7
|
|
5
8
|
module Sidekiq
|
6
9
|
class Stats
|
7
10
|
def initialize
|
8
|
-
|
11
|
+
fetch_stats_fast!
|
9
12
|
end
|
10
13
|
|
11
14
|
def processed
|
@@ -48,53 +51,78 @@ module Sidekiq
|
|
48
51
|
Sidekiq::Stats::Queues.new.lengths
|
49
52
|
end
|
50
53
|
|
51
|
-
|
52
|
-
|
54
|
+
# O(1) redis calls
|
55
|
+
def fetch_stats_fast!
|
56
|
+
pipe1_res = Sidekiq.redis { |conn|
|
53
57
|
conn.pipelined do
|
54
|
-
conn.get(
|
55
|
-
conn.get(
|
56
|
-
conn.zcard(
|
57
|
-
conn.zcard(
|
58
|
-
conn.zcard(
|
59
|
-
conn.scard(
|
60
|
-
conn.lrange(
|
61
|
-
conn.smembers('processes'.freeze)
|
62
|
-
conn.smembers('queues'.freeze)
|
58
|
+
conn.get("stat:processed")
|
59
|
+
conn.get("stat:failed")
|
60
|
+
conn.zcard("schedule")
|
61
|
+
conn.zcard("retry")
|
62
|
+
conn.zcard("dead")
|
63
|
+
conn.scard("processes")
|
64
|
+
conn.lrange("queue:default", -1, -1)
|
63
65
|
end
|
64
|
-
|
66
|
+
}
|
65
67
|
|
66
|
-
|
67
|
-
|
68
|
-
|
69
|
-
|
68
|
+
default_queue_latency = if (entry = pipe1_res[6].first)
|
69
|
+
job = begin
|
70
|
+
Sidekiq.load_json(entry)
|
71
|
+
rescue
|
72
|
+
{}
|
70
73
|
end
|
74
|
+
now = Time.now.to_f
|
75
|
+
thence = job["enqueued_at"] || now
|
76
|
+
now - thence
|
77
|
+
else
|
78
|
+
0
|
71
79
|
end
|
72
80
|
|
73
|
-
s = pipe1_res[7].size
|
74
|
-
workers_size = pipe2_res[0...s].map(&:to_i).inject(0, &:+)
|
75
|
-
enqueued = pipe2_res[s..-1].map(&:to_i).inject(0, &:+)
|
76
|
-
|
77
|
-
default_queue_latency = if (entry = pipe1_res[6].first)
|
78
|
-
Time.now.to_f - Sidekiq.load_json(entry)['enqueued_at'.freeze]
|
79
|
-
else
|
80
|
-
0
|
81
|
-
end
|
82
81
|
@stats = {
|
83
|
-
processed:
|
84
|
-
failed:
|
85
|
-
scheduled_size:
|
86
|
-
retry_size:
|
87
|
-
dead_size:
|
88
|
-
processes_size:
|
89
|
-
|
90
|
-
default_queue_latency: default_queue_latency
|
91
|
-
workers_size: workers_size,
|
92
|
-
enqueued: enqueued
|
82
|
+
processed: pipe1_res[0].to_i,
|
83
|
+
failed: pipe1_res[1].to_i,
|
84
|
+
scheduled_size: pipe1_res[2],
|
85
|
+
retry_size: pipe1_res[3],
|
86
|
+
dead_size: pipe1_res[4],
|
87
|
+
processes_size: pipe1_res[5],
|
88
|
+
|
89
|
+
default_queue_latency: default_queue_latency
|
93
90
|
}
|
94
91
|
end
|
95
92
|
|
93
|
+
# O(number of processes + number of queues) redis calls
|
94
|
+
def fetch_stats_slow!
|
95
|
+
processes = Sidekiq.redis { |conn|
|
96
|
+
conn.sscan_each("processes").to_a
|
97
|
+
}
|
98
|
+
|
99
|
+
queues = Sidekiq.redis { |conn|
|
100
|
+
conn.sscan_each("queues").to_a
|
101
|
+
}
|
102
|
+
|
103
|
+
pipe2_res = Sidekiq.redis { |conn|
|
104
|
+
conn.pipelined do
|
105
|
+
processes.each { |key| conn.hget(key, "busy") }
|
106
|
+
queues.each { |queue| conn.llen("queue:#{queue}") }
|
107
|
+
end
|
108
|
+
}
|
109
|
+
|
110
|
+
s = processes.size
|
111
|
+
workers_size = pipe2_res[0...s].sum(&:to_i)
|
112
|
+
enqueued = pipe2_res[s..-1].sum(&:to_i)
|
113
|
+
|
114
|
+
@stats[:workers_size] = workers_size
|
115
|
+
@stats[:enqueued] = enqueued
|
116
|
+
@stats
|
117
|
+
end
|
118
|
+
|
119
|
+
def fetch_stats!
|
120
|
+
fetch_stats_fast!
|
121
|
+
fetch_stats_slow!
|
122
|
+
end
|
123
|
+
|
96
124
|
def reset(*stats)
|
97
|
-
all
|
125
|
+
all = %w[failed processed]
|
98
126
|
stats = stats.empty? ? all : all & stats.flatten.compact.map(&:to_s)
|
99
127
|
|
100
128
|
mset_args = []
|
@@ -110,66 +138,62 @@ module Sidekiq
|
|
110
138
|
private
|
111
139
|
|
112
140
|
def stat(s)
|
113
|
-
@stats[s]
|
141
|
+
fetch_stats_slow! if @stats[s].nil?
|
142
|
+
@stats[s] || raise(ArgumentError, "Unknown stat #{s}")
|
114
143
|
end
|
115
144
|
|
116
145
|
class Queues
|
117
146
|
def lengths
|
118
147
|
Sidekiq.redis do |conn|
|
119
|
-
queues = conn.
|
148
|
+
queues = conn.sscan_each("queues").to_a
|
120
149
|
|
121
|
-
lengths = conn.pipelined
|
150
|
+
lengths = conn.pipelined {
|
122
151
|
queues.each do |queue|
|
123
152
|
conn.llen("queue:#{queue}")
|
124
153
|
end
|
125
|
-
|
126
|
-
|
127
|
-
i = 0
|
128
|
-
array_of_arrays = queues.inject({}) do |memo, queue|
|
129
|
-
memo[queue] = lengths[i]
|
130
|
-
i += 1
|
131
|
-
memo
|
132
|
-
end.sort_by { |_, size| size }
|
154
|
+
}
|
133
155
|
|
134
|
-
|
156
|
+
array_of_arrays = queues.zip(lengths).sort_by { |_, size| -size }
|
157
|
+
array_of_arrays.to_h
|
135
158
|
end
|
136
159
|
end
|
137
160
|
end
|
138
161
|
|
139
162
|
class History
|
140
163
|
def initialize(days_previous, start_date = nil)
|
164
|
+
# we only store five years of data in Redis
|
165
|
+
raise ArgumentError if days_previous < 1 || days_previous > (5 * 365)
|
141
166
|
@days_previous = days_previous
|
142
167
|
@start_date = start_date || Time.now.utc.to_date
|
143
168
|
end
|
144
169
|
|
145
170
|
def processed
|
146
|
-
date_stat_hash("processed")
|
171
|
+
@processed ||= date_stat_hash("processed")
|
147
172
|
end
|
148
173
|
|
149
174
|
def failed
|
150
|
-
date_stat_hash("failed")
|
175
|
+
@failed ||= date_stat_hash("failed")
|
151
176
|
end
|
152
177
|
|
153
178
|
private
|
154
179
|
|
155
180
|
def date_stat_hash(stat)
|
156
|
-
i = 0
|
157
181
|
stat_hash = {}
|
158
|
-
|
159
|
-
|
160
|
-
|
161
|
-
while i < @days_previous
|
162
|
-
date = @start_date - i
|
163
|
-
datestr = date.strftime("%Y-%m-%d".freeze)
|
164
|
-
keys << "stat:#{stat}:#{datestr}"
|
165
|
-
dates << datestr
|
166
|
-
i += 1
|
167
|
-
end
|
182
|
+
dates = @start_date.downto(@start_date - @days_previous + 1).map { |date|
|
183
|
+
date.strftime("%Y-%m-%d")
|
184
|
+
}
|
168
185
|
|
169
|
-
|
170
|
-
|
171
|
-
|
186
|
+
keys = dates.map { |datestr| "stat:#{stat}:#{datestr}" }
|
187
|
+
|
188
|
+
begin
|
189
|
+
Sidekiq.redis do |conn|
|
190
|
+
conn.mget(keys).each_with_index do |value, idx|
|
191
|
+
stat_hash[dates[idx]] = value ? value.to_i : 0
|
192
|
+
end
|
172
193
|
end
|
194
|
+
rescue Redis::CommandError
|
195
|
+
# mget will trigger a CROSSSLOT error when run against a Cluster
|
196
|
+
# TODO Someone want to add Cluster support?
|
173
197
|
end
|
174
198
|
|
175
199
|
stat_hash
|
@@ -196,13 +220,13 @@ module Sidekiq
|
|
196
220
|
# Return all known queues within Redis.
|
197
221
|
#
|
198
222
|
def self.all
|
199
|
-
Sidekiq.redis { |c| c.
|
223
|
+
Sidekiq.redis { |c| c.sscan_each("queues").to_a }.sort.map { |q| Sidekiq::Queue.new(q) }
|
200
224
|
end
|
201
225
|
|
202
226
|
attr_reader :name
|
203
227
|
|
204
|
-
def initialize(name="default")
|
205
|
-
@name = name
|
228
|
+
def initialize(name = "default")
|
229
|
+
@name = name.to_s
|
206
230
|
@rname = "queue:#{name}"
|
207
231
|
end
|
208
232
|
|
@@ -221,11 +245,14 @@ module Sidekiq
|
|
221
245
|
#
|
222
246
|
# @return Float
|
223
247
|
def latency
|
224
|
-
entry = Sidekiq.redis
|
248
|
+
entry = Sidekiq.redis { |conn|
|
225
249
|
conn.lrange(@rname, -1, -1)
|
226
|
-
|
250
|
+
}.first
|
227
251
|
return 0 unless entry
|
228
|
-
|
252
|
+
job = Sidekiq.load_json(entry)
|
253
|
+
now = Time.now.to_f
|
254
|
+
thence = job["enqueued_at"] || now
|
255
|
+
now - thence
|
229
256
|
end
|
230
257
|
|
231
258
|
def each
|
@@ -234,16 +261,16 @@ module Sidekiq
|
|
234
261
|
page = 0
|
235
262
|
page_size = 50
|
236
263
|
|
237
|
-
|
264
|
+
loop do
|
238
265
|
range_start = page * page_size - deleted_size
|
239
|
-
range_end
|
240
|
-
entries = Sidekiq.redis
|
266
|
+
range_end = range_start + page_size - 1
|
267
|
+
entries = Sidekiq.redis { |conn|
|
241
268
|
conn.lrange @rname, range_start, range_end
|
242
|
-
|
269
|
+
}
|
243
270
|
break if entries.empty?
|
244
271
|
page += 1
|
245
272
|
entries.each do |entry|
|
246
|
-
yield
|
273
|
+
yield JobRecord.new(entry, @name)
|
247
274
|
end
|
248
275
|
deleted_size = initial_size - size
|
249
276
|
end
|
@@ -253,7 +280,7 @@ module Sidekiq
|
|
253
280
|
# Find the job with the given JID within this queue.
|
254
281
|
#
|
255
282
|
# This is a slow, inefficient operation. Do not use under
|
256
|
-
# normal conditions.
|
283
|
+
# normal conditions.
|
257
284
|
def find_job(jid)
|
258
285
|
detect { |j| j.jid == jid }
|
259
286
|
end
|
@@ -261,8 +288,8 @@ module Sidekiq
|
|
261
288
|
def clear
|
262
289
|
Sidekiq.redis do |conn|
|
263
290
|
conn.multi do
|
264
|
-
conn.
|
265
|
-
conn.srem("queues"
|
291
|
+
conn.unlink(@rname)
|
292
|
+
conn.srem("queues", name)
|
266
293
|
end
|
267
294
|
end
|
268
295
|
end
|
@@ -274,114 +301,166 @@ module Sidekiq
|
|
274
301
|
# sorted set.
|
275
302
|
#
|
276
303
|
# The job should be considered immutable but may be
|
277
|
-
# removed from the queue via
|
304
|
+
# removed from the queue via JobRecord#delete.
|
278
305
|
#
|
279
|
-
class
|
306
|
+
class JobRecord
|
280
307
|
attr_reader :item
|
281
308
|
attr_reader :value
|
282
309
|
|
283
|
-
def initialize(item, queue_name=nil)
|
310
|
+
def initialize(item, queue_name = nil)
|
311
|
+
@args = nil
|
284
312
|
@value = item
|
285
|
-
@item = item.is_a?(Hash) ? item :
|
286
|
-
@queue = queue_name || @item[
|
313
|
+
@item = item.is_a?(Hash) ? item : parse(item)
|
314
|
+
@queue = queue_name || @item["queue"]
|
315
|
+
end
|
316
|
+
|
317
|
+
def parse(item)
|
318
|
+
Sidekiq.load_json(item)
|
319
|
+
rescue JSON::ParserError
|
320
|
+
# If the job payload in Redis is invalid JSON, we'll load
|
321
|
+
# the item as an empty hash and store the invalid JSON as
|
322
|
+
# the job 'args' for display in the Web UI.
|
323
|
+
@invalid = true
|
324
|
+
@args = [item]
|
325
|
+
{}
|
287
326
|
end
|
288
327
|
|
289
328
|
def klass
|
290
|
-
|
329
|
+
self["class"]
|
291
330
|
end
|
292
331
|
|
293
332
|
def display_class
|
294
333
|
# Unwrap known wrappers so they show up in a human-friendly manner in the Web UI
|
295
|
-
@klass ||=
|
296
|
-
|
297
|
-
|
298
|
-
|
299
|
-
|
300
|
-
|
301
|
-
|
302
|
-
|
303
|
-
|
304
|
-
|
305
|
-
|
306
|
-
|
307
|
-
|
308
|
-
|
309
|
-
|
310
|
-
|
334
|
+
@klass ||= self["display_class"] || begin
|
335
|
+
case klass
|
336
|
+
when /\ASidekiq::Extensions::Delayed/
|
337
|
+
safe_load(args[0], klass) do |target, method, _|
|
338
|
+
"#{target}.#{method}"
|
339
|
+
end
|
340
|
+
when "ActiveJob::QueueAdapters::SidekiqAdapter::JobWrapper"
|
341
|
+
job_class = @item["wrapped"] || args[0]
|
342
|
+
if job_class == "ActionMailer::DeliveryJob" || job_class == "ActionMailer::MailDeliveryJob"
|
343
|
+
# MailerClass#mailer_method
|
344
|
+
args[0]["arguments"][0..1].join("#")
|
345
|
+
else
|
346
|
+
job_class
|
347
|
+
end
|
348
|
+
else
|
349
|
+
klass
|
350
|
+
end
|
351
|
+
end
|
311
352
|
end
|
312
353
|
|
313
354
|
def display_args
|
314
355
|
# Unwrap known wrappers so they show up in a human-friendly manner in the Web UI
|
315
|
-
@
|
356
|
+
@display_args ||= case klass
|
316
357
|
when /\ASidekiq::Extensions::Delayed/
|
317
358
|
safe_load(args[0], args) do |_, _, arg|
|
318
359
|
arg
|
319
360
|
end
|
320
361
|
when "ActiveJob::QueueAdapters::SidekiqAdapter::JobWrapper"
|
321
|
-
job_args =
|
322
|
-
if
|
323
|
-
|
324
|
-
|
362
|
+
job_args = self["wrapped"] ? args[0]["arguments"] : []
|
363
|
+
if (self["wrapped"] || args[0]) == "ActionMailer::DeliveryJob"
|
364
|
+
# remove MailerClass, mailer_method and 'deliver_now'
|
365
|
+
job_args.drop(3)
|
366
|
+
elsif (self["wrapped"] || args[0]) == "ActionMailer::MailDeliveryJob"
|
367
|
+
# remove MailerClass, mailer_method and 'deliver_now'
|
368
|
+
job_args.drop(3).first["args"]
|
325
369
|
else
|
326
|
-
|
370
|
+
job_args
|
327
371
|
end
|
328
372
|
else
|
373
|
+
if self["encrypt"]
|
374
|
+
# no point in showing 150+ bytes of random garbage
|
375
|
+
args[-1] = "[encrypted data]"
|
376
|
+
end
|
329
377
|
args
|
330
|
-
|
378
|
+
end
|
331
379
|
end
|
332
380
|
|
333
381
|
def args
|
334
|
-
@item[
|
382
|
+
@args || @item["args"]
|
335
383
|
end
|
336
384
|
|
337
385
|
def jid
|
338
|
-
|
386
|
+
self["jid"]
|
339
387
|
end
|
340
388
|
|
341
389
|
def enqueued_at
|
342
|
-
|
390
|
+
self["enqueued_at"] ? Time.at(self["enqueued_at"]).utc : nil
|
343
391
|
end
|
344
392
|
|
345
393
|
def created_at
|
346
|
-
Time.at(
|
394
|
+
Time.at(self["created_at"] || self["enqueued_at"] || 0).utc
|
395
|
+
end
|
396
|
+
|
397
|
+
def tags
|
398
|
+
self["tags"] || []
|
347
399
|
end
|
348
400
|
|
349
|
-
def
|
350
|
-
|
401
|
+
def error_backtrace
|
402
|
+
# Cache nil values
|
403
|
+
if defined?(@error_backtrace)
|
404
|
+
@error_backtrace
|
405
|
+
else
|
406
|
+
value = self["error_backtrace"]
|
407
|
+
@error_backtrace = value && uncompress_backtrace(value)
|
408
|
+
end
|
351
409
|
end
|
352
410
|
|
411
|
+
attr_reader :queue
|
412
|
+
|
353
413
|
def latency
|
354
|
-
Time.now.to_f
|
414
|
+
now = Time.now.to_f
|
415
|
+
now - (@item["enqueued_at"] || @item["created_at"] || now)
|
355
416
|
end
|
356
417
|
|
357
418
|
##
|
358
419
|
# Remove this job from the queue.
|
359
420
|
def delete
|
360
|
-
count = Sidekiq.redis
|
421
|
+
count = Sidekiq.redis { |conn|
|
361
422
|
conn.lrem("queue:#{@queue}", 1, @value)
|
362
|
-
|
423
|
+
}
|
363
424
|
count != 0
|
364
425
|
end
|
365
426
|
|
366
427
|
def [](name)
|
367
|
-
|
428
|
+
# nil will happen if the JSON fails to parse.
|
429
|
+
# We don't guarantee Sidekiq will work with bad job JSON but we should
|
430
|
+
# make a best effort to minimize the damage.
|
431
|
+
@item ? @item[name] : nil
|
368
432
|
end
|
369
433
|
|
370
434
|
private
|
371
435
|
|
372
436
|
def safe_load(content, default)
|
373
|
-
|
374
|
-
|
375
|
-
|
376
|
-
|
377
|
-
|
378
|
-
|
379
|
-
|
437
|
+
yield(*YAML.load(content))
|
438
|
+
rescue => ex
|
439
|
+
# #1761 in dev mode, it's possible to have jobs enqueued which haven't been loaded into
|
440
|
+
# memory yet so the YAML can't be loaded.
|
441
|
+
Sidekiq.logger.warn "Unable to load YAML: #{ex.message}" unless Sidekiq.options[:environment] == "development"
|
442
|
+
default
|
443
|
+
end
|
444
|
+
|
445
|
+
def uncompress_backtrace(backtrace)
|
446
|
+
if backtrace.is_a?(Array)
|
447
|
+
# Handle old jobs with raw Array backtrace format
|
448
|
+
backtrace
|
449
|
+
else
|
450
|
+
decoded = Base64.decode64(backtrace)
|
451
|
+
uncompressed = Zlib::Inflate.inflate(decoded)
|
452
|
+
begin
|
453
|
+
Sidekiq.load_json(uncompressed)
|
454
|
+
rescue
|
455
|
+
# Handle old jobs with marshalled backtrace format
|
456
|
+
# TODO Remove in 7.x
|
457
|
+
Marshal.load(uncompressed)
|
458
|
+
end
|
380
459
|
end
|
381
460
|
end
|
382
461
|
end
|
383
462
|
|
384
|
-
class SortedEntry <
|
463
|
+
class SortedEntry < JobRecord
|
385
464
|
attr_reader :score
|
386
465
|
attr_reader :parent
|
387
466
|
|
@@ -404,8 +483,9 @@ module Sidekiq
|
|
404
483
|
end
|
405
484
|
|
406
485
|
def reschedule(at)
|
407
|
-
|
408
|
-
|
486
|
+
Sidekiq.redis do |conn|
|
487
|
+
conn.zincrby(@parent.name, at.to_f - @score, Sidekiq.dump_json(@item))
|
488
|
+
end
|
409
489
|
end
|
410
490
|
|
411
491
|
def add_to_queue
|
@@ -416,10 +496,9 @@ module Sidekiq
|
|
416
496
|
end
|
417
497
|
|
418
498
|
def retry
|
419
|
-
raise "Retry not available on jobs which have not failed" unless item["failed_at"]
|
420
499
|
remove_job do |message|
|
421
500
|
msg = Sidekiq.load_json(message)
|
422
|
-
msg[
|
501
|
+
msg["retry_count"] -= 1 if msg["retry_count"]
|
423
502
|
Sidekiq::Client.push(msg)
|
424
503
|
end
|
425
504
|
end
|
@@ -427,56 +506,50 @@ module Sidekiq
|
|
427
506
|
##
|
428
507
|
# Place job in the dead set
|
429
508
|
def kill
|
430
|
-
raise 'Kill not available on jobs which have not failed' unless item['failed_at']
|
431
509
|
remove_job do |message|
|
432
|
-
|
433
|
-
now = Time.now.to_f
|
434
|
-
Sidekiq.redis do |conn|
|
435
|
-
conn.multi do
|
436
|
-
conn.zadd('dead', now, message)
|
437
|
-
conn.zremrangebyscore('dead', '-inf', now - DeadSet.timeout)
|
438
|
-
conn.zremrangebyrank('dead', 0, - DeadSet.max_jobs)
|
439
|
-
end
|
440
|
-
end
|
510
|
+
DeadSet.new.kill(message)
|
441
511
|
end
|
442
512
|
end
|
443
513
|
|
514
|
+
def error?
|
515
|
+
!!item["error_class"]
|
516
|
+
end
|
517
|
+
|
444
518
|
private
|
445
519
|
|
446
520
|
def remove_job
|
447
521
|
Sidekiq.redis do |conn|
|
448
|
-
results = conn.multi
|
522
|
+
results = conn.multi {
|
449
523
|
conn.zrangebyscore(parent.name, score, score)
|
450
524
|
conn.zremrangebyscore(parent.name, score, score)
|
451
|
-
|
525
|
+
}.first
|
452
526
|
|
453
527
|
if results.size == 1
|
454
528
|
yield results.first
|
455
529
|
else
|
456
530
|
# multiple jobs with the same score
|
457
531
|
# find the one with the right JID and push it
|
458
|
-
|
532
|
+
matched, nonmatched = results.partition { |message|
|
459
533
|
if message.index(jid)
|
460
534
|
msg = Sidekiq.load_json(message)
|
461
|
-
msg[
|
535
|
+
msg["jid"] == jid
|
462
536
|
else
|
463
537
|
false
|
464
538
|
end
|
465
|
-
|
539
|
+
}
|
466
540
|
|
467
|
-
msg =
|
541
|
+
msg = matched.first
|
468
542
|
yield msg if msg
|
469
543
|
|
470
544
|
# push the rest back onto the sorted set
|
471
545
|
conn.multi do
|
472
|
-
|
546
|
+
nonmatched.each do |message|
|
473
547
|
conn.zadd(parent.name, score.to_f.to_s, message)
|
474
548
|
end
|
475
549
|
end
|
476
550
|
end
|
477
551
|
end
|
478
552
|
end
|
479
|
-
|
480
553
|
end
|
481
554
|
|
482
555
|
class SortedSet
|
@@ -493,16 +566,26 @@ module Sidekiq
|
|
493
566
|
Sidekiq.redis { |c| c.zcard(name) }
|
494
567
|
end
|
495
568
|
|
569
|
+
def scan(match, count = 100)
|
570
|
+
return to_enum(:scan, match, count) unless block_given?
|
571
|
+
|
572
|
+
match = "*#{match}*" unless match.include?("*")
|
573
|
+
Sidekiq.redis do |conn|
|
574
|
+
conn.zscan_each(name, match: match, count: count) do |entry, score|
|
575
|
+
yield SortedEntry.new(self, score, entry)
|
576
|
+
end
|
577
|
+
end
|
578
|
+
end
|
579
|
+
|
496
580
|
def clear
|
497
581
|
Sidekiq.redis do |conn|
|
498
|
-
conn.
|
582
|
+
conn.unlink(name)
|
499
583
|
end
|
500
584
|
end
|
501
585
|
alias_method :💣, :clear
|
502
586
|
end
|
503
587
|
|
504
588
|
class JobSet < SortedSet
|
505
|
-
|
506
589
|
def schedule(timestamp, message)
|
507
590
|
Sidekiq.redis do |conn|
|
508
591
|
conn.zadd(name, timestamp.to_f.to_s, Sidekiq.dump_json(message))
|
@@ -515,44 +598,55 @@ module Sidekiq
|
|
515
598
|
page = -1
|
516
599
|
page_size = 50
|
517
600
|
|
518
|
-
|
601
|
+
loop do
|
519
602
|
range_start = page * page_size + offset_size
|
520
|
-
range_end
|
521
|
-
elements = Sidekiq.redis
|
603
|
+
range_end = range_start + page_size - 1
|
604
|
+
elements = Sidekiq.redis { |conn|
|
522
605
|
conn.zrange name, range_start, range_end, with_scores: true
|
523
|
-
|
606
|
+
}
|
524
607
|
break if elements.empty?
|
525
608
|
page -= 1
|
526
|
-
elements.
|
609
|
+
elements.reverse_each do |element, score|
|
527
610
|
yield SortedEntry.new(self, score, element)
|
528
611
|
end
|
529
612
|
offset_size = initial_size - @_size
|
530
613
|
end
|
531
614
|
end
|
532
615
|
|
616
|
+
##
|
617
|
+
# Fetch jobs that match a given time or Range. Job ID is an
|
618
|
+
# optional second argument.
|
533
619
|
def fetch(score, jid = nil)
|
534
|
-
|
535
|
-
|
536
|
-
|
537
|
-
|
538
|
-
elements.inject([]) do |result, element|
|
539
|
-
entry = SortedEntry.new(self, score, element)
|
540
|
-
if jid
|
541
|
-
result << entry if entry.jid == jid
|
620
|
+
begin_score, end_score =
|
621
|
+
if score.is_a?(Range)
|
622
|
+
[score.first, score.last]
|
542
623
|
else
|
543
|
-
|
624
|
+
[score, score]
|
544
625
|
end
|
545
|
-
|
626
|
+
|
627
|
+
elements = Sidekiq.redis { |conn|
|
628
|
+
conn.zrangebyscore(name, begin_score, end_score, with_scores: true)
|
629
|
+
}
|
630
|
+
|
631
|
+
elements.each_with_object([]) do |element, result|
|
632
|
+
data, job_score = element
|
633
|
+
entry = SortedEntry.new(self, job_score, data)
|
634
|
+
result << entry if jid.nil? || entry.jid == jid
|
546
635
|
end
|
547
636
|
end
|
548
637
|
|
549
638
|
##
|
550
639
|
# Find the job with the given JID within this sorted set.
|
551
|
-
#
|
552
|
-
# This is a slow, inefficient operation. Do not use under
|
553
|
-
# normal conditions. Sidekiq Pro contains a faster version.
|
640
|
+
# This is a slower O(n) operation. Do not use for app logic.
|
554
641
|
def find_job(jid)
|
555
|
-
|
642
|
+
Sidekiq.redis do |conn|
|
643
|
+
conn.zscan_each(name, match: "*#{jid}*", count: 100) do |entry, score|
|
644
|
+
job = JSON.parse(entry)
|
645
|
+
matched = job["jid"] == jid
|
646
|
+
return SortedEntry.new(self, score, entry) if matched
|
647
|
+
end
|
648
|
+
end
|
649
|
+
nil
|
556
650
|
end
|
557
651
|
|
558
652
|
def delete_by_value(name, value)
|
@@ -567,13 +661,14 @@ module Sidekiq
|
|
567
661
|
Sidekiq.redis do |conn|
|
568
662
|
elements = conn.zrangebyscore(name, score, score)
|
569
663
|
elements.each do |element|
|
570
|
-
|
571
|
-
|
572
|
-
|
573
|
-
|
574
|
-
|
664
|
+
if element.index(jid)
|
665
|
+
message = Sidekiq.load_json(element)
|
666
|
+
if message["jid"] == jid
|
667
|
+
ret = conn.zrem(name, element)
|
668
|
+
@_size -= 1 if ret
|
669
|
+
break ret
|
670
|
+
end
|
575
671
|
end
|
576
|
-
false
|
577
672
|
end
|
578
673
|
end
|
579
674
|
end
|
@@ -585,17 +680,17 @@ module Sidekiq
|
|
585
680
|
# Allows enumeration of scheduled jobs within Sidekiq.
|
586
681
|
# Based on this, you can search/filter for jobs. Here's an
|
587
682
|
# example where I'm selecting all jobs of a certain type
|
588
|
-
# and deleting them from the
|
683
|
+
# and deleting them from the schedule queue.
|
589
684
|
#
|
590
685
|
# r = Sidekiq::ScheduledSet.new
|
591
|
-
# r.select do |
|
592
|
-
#
|
593
|
-
#
|
594
|
-
#
|
686
|
+
# r.select do |scheduled|
|
687
|
+
# scheduled.klass == 'Sidekiq::Extensions::DelayedClass' &&
|
688
|
+
# scheduled.args[0] == 'User' &&
|
689
|
+
# scheduled.args[1] == 'setup_new_subscriber'
|
595
690
|
# end.map(&:delete)
|
596
691
|
class ScheduledSet < JobSet
|
597
692
|
def initialize
|
598
|
-
super
|
693
|
+
super "schedule"
|
599
694
|
end
|
600
695
|
end
|
601
696
|
|
@@ -613,13 +708,15 @@ module Sidekiq
|
|
613
708
|
# end.map(&:delete)
|
614
709
|
class RetrySet < JobSet
|
615
710
|
def initialize
|
616
|
-
super
|
711
|
+
super "retry"
|
617
712
|
end
|
618
713
|
|
619
714
|
def retry_all
|
620
|
-
while size > 0
|
621
|
-
|
622
|
-
|
715
|
+
each(&:retry) while size > 0
|
716
|
+
end
|
717
|
+
|
718
|
+
def kill_all
|
719
|
+
each(&:kill) while size > 0
|
623
720
|
end
|
624
721
|
end
|
625
722
|
|
@@ -628,13 +725,32 @@ module Sidekiq
|
|
628
725
|
#
|
629
726
|
class DeadSet < JobSet
|
630
727
|
def initialize
|
631
|
-
super
|
728
|
+
super "dead"
|
632
729
|
end
|
633
730
|
|
634
|
-
def
|
635
|
-
|
636
|
-
|
731
|
+
def kill(message, opts = {})
|
732
|
+
now = Time.now.to_f
|
733
|
+
Sidekiq.redis do |conn|
|
734
|
+
conn.multi do
|
735
|
+
conn.zadd(name, now.to_s, message)
|
736
|
+
conn.zremrangebyscore(name, "-inf", now - self.class.timeout)
|
737
|
+
conn.zremrangebyrank(name, 0, - self.class.max_jobs)
|
738
|
+
end
|
739
|
+
end
|
740
|
+
|
741
|
+
if opts[:notify_failure] != false
|
742
|
+
job = Sidekiq.load_json(message)
|
743
|
+
r = RuntimeError.new("Job killed by API")
|
744
|
+
r.set_backtrace(caller)
|
745
|
+
Sidekiq.death_handlers.each do |handle|
|
746
|
+
handle.call(job, r)
|
747
|
+
end
|
637
748
|
end
|
749
|
+
true
|
750
|
+
end
|
751
|
+
|
752
|
+
def retry_all
|
753
|
+
each(&:retry) while size > 0
|
638
754
|
end
|
639
755
|
|
640
756
|
def self.max_jobs
|
@@ -648,7 +764,7 @@ module Sidekiq
|
|
648
764
|
|
649
765
|
##
|
650
766
|
# Enumerates the set of Sidekiq processes which are actively working
|
651
|
-
# right now. Each process
|
767
|
+
# right now. Each process sends a heartbeat to Redis every 5 seconds
|
652
768
|
# so this set should be relatively accurate, barring network partitions.
|
653
769
|
#
|
654
770
|
# Yields a Sidekiq::Process.
|
@@ -656,54 +772,60 @@ module Sidekiq
|
|
656
772
|
class ProcessSet
|
657
773
|
include Enumerable
|
658
774
|
|
659
|
-
def initialize(clean_plz=true)
|
660
|
-
|
775
|
+
def initialize(clean_plz = true)
|
776
|
+
cleanup if clean_plz
|
661
777
|
end
|
662
778
|
|
663
779
|
# Cleans up dead processes recorded in Redis.
|
664
780
|
# Returns the number of processes cleaned.
|
665
|
-
def
|
781
|
+
def cleanup
|
666
782
|
count = 0
|
667
783
|
Sidekiq.redis do |conn|
|
668
|
-
procs = conn.
|
669
|
-
heartbeats = conn.pipelined
|
784
|
+
procs = conn.sscan_each("processes").to_a.sort
|
785
|
+
heartbeats = conn.pipelined {
|
670
786
|
procs.each do |key|
|
671
|
-
conn.hget(key,
|
787
|
+
conn.hget(key, "info")
|
672
788
|
end
|
673
|
-
|
789
|
+
}
|
674
790
|
|
675
791
|
# the hash named key has an expiry of 60 seconds.
|
676
792
|
# if it's not found, that means the process has not reported
|
677
793
|
# in to Redis and probably died.
|
678
|
-
to_prune =
|
679
|
-
|
680
|
-
|
681
|
-
|
682
|
-
count = conn.srem('processes', to_prune) unless to_prune.empty?
|
794
|
+
to_prune = procs.select.with_index { |proc, i|
|
795
|
+
heartbeats[i].nil?
|
796
|
+
}
|
797
|
+
count = conn.srem("processes", to_prune) unless to_prune.empty?
|
683
798
|
end
|
684
799
|
count
|
685
800
|
end
|
686
801
|
|
687
802
|
def each
|
688
|
-
|
803
|
+
result = Sidekiq.redis { |conn|
|
804
|
+
procs = conn.sscan_each("processes").to_a.sort
|
689
805
|
|
690
|
-
Sidekiq.redis do |conn|
|
691
806
|
# We're making a tradeoff here between consuming more memory instead of
|
692
807
|
# making more roundtrips to Redis, but if you have hundreds or thousands of workers,
|
693
808
|
# you'll be happier this way
|
694
|
-
|
809
|
+
conn.pipelined do
|
695
810
|
procs.each do |key|
|
696
|
-
conn.hmget(key,
|
811
|
+
conn.hmget(key, "info", "busy", "beat", "quiet", "rss", "rtt_us")
|
697
812
|
end
|
698
813
|
end
|
814
|
+
}
|
699
815
|
|
700
|
-
|
701
|
-
|
702
|
-
|
703
|
-
|
704
|
-
|
816
|
+
result.each do |info, busy, at_s, quiet, rss, rtt|
|
817
|
+
# If a process is stopped between when we query Redis for `procs` and
|
818
|
+
# when we query for `result`, we will have an item in `result` that is
|
819
|
+
# composed of `nil` values.
|
820
|
+
next if info.nil?
|
705
821
|
|
706
|
-
|
822
|
+
hash = Sidekiq.load_json(info)
|
823
|
+
yield Process.new(hash.merge("busy" => busy.to_i,
|
824
|
+
"beat" => at_s.to_f,
|
825
|
+
"quiet" => quiet,
|
826
|
+
"rss" => rss.to_i,
|
827
|
+
"rtt_us" => rtt.to_i))
|
828
|
+
end
|
707
829
|
end
|
708
830
|
|
709
831
|
# This method is not guaranteed accurate since it does not prune the set
|
@@ -711,7 +833,31 @@ module Sidekiq
|
|
711
833
|
# contains Sidekiq processes which have sent a heartbeat within the last
|
712
834
|
# 60 seconds.
|
713
835
|
def size
|
714
|
-
Sidekiq.redis { |conn| conn.scard(
|
836
|
+
Sidekiq.redis { |conn| conn.scard("processes") }
|
837
|
+
end
|
838
|
+
|
839
|
+
# Total number of threads available to execute jobs.
|
840
|
+
# For Sidekiq Enterprise customers this number (in production) must be
|
841
|
+
# less than or equal to your licensed concurrency.
|
842
|
+
def total_concurrency
|
843
|
+
sum { |x| x["concurrency"].to_i }
|
844
|
+
end
|
845
|
+
|
846
|
+
def total_rss_in_kb
|
847
|
+
sum { |x| x["rss"].to_i }
|
848
|
+
end
|
849
|
+
alias_method :total_rss, :total_rss_in_kb
|
850
|
+
|
851
|
+
# Returns the identity of the current cluster leader or "" if no leader.
|
852
|
+
# This is a Sidekiq Enterprise feature, will always return "" in Sidekiq
|
853
|
+
# or Sidekiq Pro.
|
854
|
+
def leader
|
855
|
+
@leader ||= begin
|
856
|
+
x = Sidekiq.redis { |c| c.get("dear-leader") }
|
857
|
+
# need a non-falsy value so we can memoize
|
858
|
+
x ||= ""
|
859
|
+
x
|
860
|
+
end
|
715
861
|
end
|
716
862
|
end
|
717
863
|
|
@@ -736,31 +882,39 @@ module Sidekiq
|
|
736
882
|
end
|
737
883
|
|
738
884
|
def tag
|
739
|
-
self[
|
885
|
+
self["tag"]
|
740
886
|
end
|
741
887
|
|
742
888
|
def labels
|
743
|
-
Array(self[
|
889
|
+
Array(self["labels"])
|
744
890
|
end
|
745
891
|
|
746
892
|
def [](key)
|
747
893
|
@attribs[key]
|
748
894
|
end
|
749
895
|
|
896
|
+
def identity
|
897
|
+
self["identity"]
|
898
|
+
end
|
899
|
+
|
900
|
+
def queues
|
901
|
+
self["queues"]
|
902
|
+
end
|
903
|
+
|
750
904
|
def quiet!
|
751
|
-
signal(
|
905
|
+
signal("TSTP")
|
752
906
|
end
|
753
907
|
|
754
908
|
def stop!
|
755
|
-
signal(
|
909
|
+
signal("TERM")
|
756
910
|
end
|
757
911
|
|
758
912
|
def dump_threads
|
759
|
-
signal(
|
913
|
+
signal("TTIN")
|
760
914
|
end
|
761
915
|
|
762
916
|
def stopping?
|
763
|
-
self[
|
917
|
+
self["quiet"] == "true"
|
764
918
|
end
|
765
919
|
|
766
920
|
private
|
@@ -774,15 +928,11 @@ module Sidekiq
|
|
774
928
|
end
|
775
929
|
end
|
776
930
|
end
|
777
|
-
|
778
|
-
def identity
|
779
|
-
self['identity']
|
780
|
-
end
|
781
931
|
end
|
782
932
|
|
783
933
|
##
|
784
|
-
#
|
785
|
-
#
|
934
|
+
# The WorkSet stores the work being done by this Sidekiq cluster.
|
935
|
+
# It tracks the process and thread working on each job.
|
786
936
|
#
|
787
937
|
# WARNING WARNING WARNING
|
788
938
|
#
|
@@ -790,33 +940,40 @@ module Sidekiq
|
|
790
940
|
# If you call #size => 5 and then expect #each to be
|
791
941
|
# called 5 times, you're going to have a bad time.
|
792
942
|
#
|
793
|
-
#
|
794
|
-
#
|
795
|
-
#
|
943
|
+
# works = Sidekiq::WorkSet.new
|
944
|
+
# works.size => 2
|
945
|
+
# works.each do |process_id, thread_id, work|
|
796
946
|
# # process_id is a unique identifier per Sidekiq process
|
797
947
|
# # thread_id is a unique identifier per thread
|
798
948
|
# # work is a Hash which looks like:
|
799
|
-
# # { 'queue' => name, 'run_at' => timestamp, 'payload' =>
|
949
|
+
# # { 'queue' => name, 'run_at' => timestamp, 'payload' => job_hash }
|
800
950
|
# # run_at is an epoch Integer.
|
801
951
|
# end
|
802
952
|
#
|
803
|
-
class
|
953
|
+
class WorkSet
|
804
954
|
include Enumerable
|
805
955
|
|
806
|
-
def each
|
956
|
+
def each(&block)
|
957
|
+
results = []
|
807
958
|
Sidekiq.redis do |conn|
|
808
|
-
procs = conn.
|
959
|
+
procs = conn.sscan_each("processes").to_a
|
809
960
|
procs.sort.each do |key|
|
810
|
-
valid, workers = conn.pipelined
|
811
|
-
conn.exists(key)
|
961
|
+
valid, workers = conn.pipelined {
|
962
|
+
conn.exists?(key)
|
812
963
|
conn.hgetall("#{key}:workers")
|
813
|
-
|
964
|
+
}
|
814
965
|
next unless valid
|
815
966
|
workers.each_pair do |tid, json|
|
816
|
-
|
967
|
+
hsh = Sidekiq.load_json(json)
|
968
|
+
p = hsh["payload"]
|
969
|
+
# avoid breaking API, this is a side effect of the JSON optimization in #4316
|
970
|
+
hsh["payload"] = Sidekiq.load_json(p) if p.is_a?(String)
|
971
|
+
results << [key, tid, hsh]
|
817
972
|
end
|
818
973
|
end
|
819
974
|
end
|
975
|
+
|
976
|
+
results.sort_by { |(_, _, hsh)| hsh["run_at"] }.each(&block)
|
820
977
|
end
|
821
978
|
|
822
979
|
# Note that #size is only as accurate as Sidekiq's heartbeat,
|
@@ -827,18 +984,21 @@ module Sidekiq
|
|
827
984
|
# which can easily get out of sync with crashy processes.
|
828
985
|
def size
|
829
986
|
Sidekiq.redis do |conn|
|
830
|
-
procs = conn.
|
987
|
+
procs = conn.sscan_each("processes").to_a
|
831
988
|
if procs.empty?
|
832
989
|
0
|
833
990
|
else
|
834
|
-
conn.pipelined
|
991
|
+
conn.pipelined {
|
835
992
|
procs.each do |key|
|
836
|
-
conn.hget(key,
|
993
|
+
conn.hget(key, "busy")
|
837
994
|
end
|
838
|
-
|
995
|
+
}.sum(&:to_i)
|
839
996
|
end
|
840
997
|
end
|
841
998
|
end
|
842
999
|
end
|
843
|
-
|
1000
|
+
# Since "worker" is a nebulous term, we've deprecated the use of this class name.
|
1001
|
+
# Is "worker" a process, a type of job, a thread? Undefined!
|
1002
|
+
# WorkSet better describes the data.
|
1003
|
+
Workers = WorkSet
|
844
1004
|
end
|