sidekiq 5.2.4 → 6.0.1
Sign up to get free protection for your applications and to get access to all the features.
Potentially problematic release.
This version of sidekiq might be problematic. Click here for more details.
- checksums.yaml +4 -4
- data/.circleci/config.yml +82 -0
- data/.gitignore +0 -2
- data/.standard.yml +20 -0
- data/6.0-Upgrade.md +72 -0
- data/COMM-LICENSE +11 -9
- data/Changes.md +129 -0
- data/Ent-2.0-Upgrade.md +37 -0
- data/Ent-Changes.md +32 -1
- data/Gemfile +12 -17
- data/Gemfile.lock +196 -0
- data/Pro-5.0-Upgrade.md +25 -0
- data/Pro-Changes.md +26 -2
- data/README.md +18 -31
- data/Rakefile +5 -4
- data/bin/sidekiqload +33 -25
- data/bin/sidekiqmon +8 -0
- data/lib/generators/sidekiq/templates/worker_test.rb.erb +1 -1
- data/lib/generators/sidekiq/worker_generator.rb +20 -12
- data/lib/sidekiq.rb +62 -43
- data/lib/sidekiq/api.rb +196 -175
- data/lib/sidekiq/cli.rb +118 -178
- data/lib/sidekiq/client.rb +51 -46
- data/lib/sidekiq/delay.rb +5 -6
- data/lib/sidekiq/exception_handler.rb +10 -12
- data/lib/sidekiq/extensions/action_mailer.rb +10 -20
- data/lib/sidekiq/extensions/active_record.rb +9 -7
- data/lib/sidekiq/extensions/class_methods.rb +9 -7
- data/lib/sidekiq/extensions/generic_proxy.rb +4 -4
- data/lib/sidekiq/fetch.rb +11 -12
- data/lib/sidekiq/job_logger.rb +45 -7
- data/lib/sidekiq/job_retry.rb +67 -58
- data/lib/sidekiq/launcher.rb +57 -51
- data/lib/sidekiq/logger.rb +165 -0
- data/lib/sidekiq/manager.rb +7 -9
- data/lib/sidekiq/middleware/chain.rb +14 -4
- data/lib/sidekiq/middleware/i18n.rb +5 -7
- data/lib/sidekiq/monitor.rb +148 -0
- data/lib/sidekiq/paginator.rb +18 -14
- data/lib/sidekiq/processor.rb +96 -66
- data/lib/sidekiq/rails.rb +23 -29
- data/lib/sidekiq/redis_connection.rb +31 -37
- data/lib/sidekiq/scheduled.rb +28 -29
- data/lib/sidekiq/testing.rb +34 -23
- data/lib/sidekiq/testing/inline.rb +2 -1
- data/lib/sidekiq/util.rb +17 -14
- data/lib/sidekiq/version.rb +2 -1
- data/lib/sidekiq/web.rb +41 -49
- data/lib/sidekiq/web/action.rb +14 -10
- data/lib/sidekiq/web/application.rb +63 -64
- data/lib/sidekiq/web/helpers.rb +92 -68
- data/lib/sidekiq/web/router.rb +17 -14
- data/lib/sidekiq/worker.rb +129 -97
- data/sidekiq.gemspec +16 -16
- data/web/assets/javascripts/dashboard.js +4 -23
- data/web/assets/stylesheets/application-dark.css +125 -0
- data/web/assets/stylesheets/application.css +9 -0
- data/web/assets/stylesheets/bootstrap.css +1 -1
- data/web/locales/ja.yml +2 -1
- data/web/views/_job_info.erb +2 -1
- data/web/views/busy.erb +4 -1
- data/web/views/dead.erb +2 -2
- data/web/views/layout.erb +1 -0
- data/web/views/morgue.erb +4 -1
- data/web/views/queue.erb +10 -1
- data/web/views/queues.erb +1 -1
- data/web/views/retries.erb +4 -1
- data/web/views/retry.erb +2 -2
- data/web/views/scheduled.erb +4 -1
- metadata +20 -30
- data/.travis.yml +0 -17
- data/Appraisals +0 -9
- data/bin/sidekiqctl +0 -237
- data/gemfiles/rails_4.gemfile +0 -31
- data/gemfiles/rails_5.gemfile +0 -31
- data/lib/sidekiq/core_ext.rb +0 -1
- data/lib/sidekiq/logging.rb +0 -122
- data/lib/sidekiq/middleware/server/active_record.rb +0 -23
data/bin/sidekiqmon
ADDED
@@ -1,6 +1,6 @@
|
|
1
1
|
require 'test_helper'
|
2
2
|
<% module_namespacing do -%>
|
3
|
-
class <%= class_name %>WorkerTest <
|
3
|
+
class <%= class_name %>WorkerTest < Minitest::Test
|
4
4
|
def test_example
|
5
5
|
skip "add some examples to (or delete) #{__FILE__}"
|
6
6
|
end
|
@@ -1,21 +1,23 @@
|
|
1
|
-
require
|
1
|
+
require "rails/generators/named_base"
|
2
2
|
|
3
3
|
module Sidekiq
|
4
4
|
module Generators # :nodoc:
|
5
5
|
class WorkerGenerator < ::Rails::Generators::NamedBase # :nodoc:
|
6
|
-
desc
|
6
|
+
desc "This generator creates a Sidekiq Worker in app/workers and a corresponding test"
|
7
7
|
|
8
|
-
check_class_collision suffix:
|
8
|
+
check_class_collision suffix: "Worker"
|
9
9
|
|
10
10
|
def self.default_generator_root
|
11
11
|
File.dirname(__FILE__)
|
12
12
|
end
|
13
13
|
|
14
14
|
def create_worker_file
|
15
|
-
template
|
15
|
+
template "worker.rb.erb", File.join("app/workers", class_path, "#{file_name}_worker.rb")
|
16
16
|
end
|
17
17
|
|
18
18
|
def create_test_file
|
19
|
+
return unless test_framework
|
20
|
+
|
19
21
|
if defined?(RSpec)
|
20
22
|
create_worker_spec
|
21
23
|
else
|
@@ -27,23 +29,29 @@ module Sidekiq
|
|
27
29
|
|
28
30
|
def create_worker_spec
|
29
31
|
template_file = File.join(
|
30
|
-
|
31
|
-
|
32
|
-
|
32
|
+
"spec/workers",
|
33
|
+
class_path,
|
34
|
+
"#{file_name}_worker_spec.rb"
|
33
35
|
)
|
34
|
-
template
|
36
|
+
template "worker_spec.rb.erb", template_file
|
35
37
|
end
|
36
38
|
|
37
39
|
def create_worker_test
|
38
40
|
template_file = File.join(
|
39
|
-
|
40
|
-
|
41
|
-
|
41
|
+
"test/workers",
|
42
|
+
class_path,
|
43
|
+
"#{file_name}_worker_test.rb"
|
42
44
|
)
|
43
|
-
template
|
45
|
+
template "worker_test.rb.erb", template_file
|
44
46
|
end
|
45
47
|
|
48
|
+
def file_name
|
49
|
+
@_file_name ||= super.sub(/_?worker\z/i, "")
|
50
|
+
end
|
46
51
|
|
52
|
+
def test_framework
|
53
|
+
::Rails.application.config.generators.options[:rails][:test_framework]
|
54
|
+
end
|
47
55
|
end
|
48
56
|
end
|
49
57
|
end
|
data/lib/sidekiq.rb
CHANGED
@@ -1,27 +1,27 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
-
require
|
4
|
-
fail "Sidekiq #{Sidekiq::VERSION} does not support Ruby versions below 2.
|
3
|
+
require "sidekiq/version"
|
4
|
+
fail "Sidekiq #{Sidekiq::VERSION} does not support Ruby versions below 2.5.0." if RUBY_PLATFORM != "java" && Gem::Version.new(RUBY_VERSION) < Gem::Version.new("2.5.0")
|
5
5
|
|
6
|
-
require
|
7
|
-
require
|
8
|
-
require
|
9
|
-
require
|
10
|
-
require
|
6
|
+
require "sidekiq/logger"
|
7
|
+
require "sidekiq/client"
|
8
|
+
require "sidekiq/worker"
|
9
|
+
require "sidekiq/redis_connection"
|
10
|
+
require "sidekiq/delay"
|
11
11
|
|
12
|
-
require
|
12
|
+
require "json"
|
13
13
|
|
14
14
|
module Sidekiq
|
15
|
-
NAME =
|
16
|
-
LICENSE =
|
15
|
+
NAME = "Sidekiq"
|
16
|
+
LICENSE = "See LICENSE and the LGPL-3.0 for licensing details."
|
17
17
|
|
18
18
|
DEFAULTS = {
|
19
19
|
queues: [],
|
20
20
|
labels: [],
|
21
21
|
concurrency: 10,
|
22
|
-
require:
|
22
|
+
require: ".",
|
23
23
|
environment: nil,
|
24
|
-
timeout:
|
24
|
+
timeout: 25,
|
25
25
|
poll_interval_average: nil,
|
26
26
|
average_scheduled_poll_interval: 5,
|
27
27
|
error_handlers: [],
|
@@ -38,8 +38,8 @@ module Sidekiq
|
|
38
38
|
}
|
39
39
|
|
40
40
|
DEFAULT_WORKER_OPTIONS = {
|
41
|
-
|
42
|
-
|
41
|
+
"retry" => true,
|
42
|
+
"queue" => "default",
|
43
43
|
}
|
44
44
|
|
45
45
|
FAKE_INFO = {
|
@@ -47,7 +47,7 @@ module Sidekiq
|
|
47
47
|
"uptime_in_days" => "9999",
|
48
48
|
"connected_clients" => "9999",
|
49
49
|
"used_memory_human" => "9P",
|
50
|
-
"used_memory_peak_human" => "9P"
|
50
|
+
"used_memory_peak_human" => "9P",
|
51
51
|
}
|
52
52
|
|
53
53
|
def self.❨╯°□°❩╯︵┻━┻
|
@@ -96,9 +96,13 @@ module Sidekiq
|
|
96
96
|
begin
|
97
97
|
yield conn
|
98
98
|
rescue Redis::CommandError => ex
|
99
|
-
#2550 Failover can cause the server to become a
|
100
|
-
# to disconnect and reopen the socket to get back to the
|
101
|
-
|
99
|
+
# 2550 Failover can cause the server to become a replica, need
|
100
|
+
# to disconnect and reopen the socket to get back to the primary.
|
101
|
+
if retryable && ex.message =~ /READONLY/
|
102
|
+
conn.disconnect!
|
103
|
+
retryable = false
|
104
|
+
retry
|
105
|
+
end
|
102
106
|
raise
|
103
107
|
end
|
104
108
|
end
|
@@ -106,19 +110,17 @@ module Sidekiq
|
|
106
110
|
|
107
111
|
def self.redis_info
|
108
112
|
redis do |conn|
|
109
|
-
|
110
|
-
|
111
|
-
|
112
|
-
|
113
|
-
|
114
|
-
|
115
|
-
conn.info
|
116
|
-
end
|
117
|
-
rescue Redis::CommandError => ex
|
118
|
-
#2850 return fake version when INFO command has (probably) been renamed
|
119
|
-
raise unless ex.message =~ /unknown command/
|
120
|
-
FAKE_INFO
|
113
|
+
# admin commands can't go through redis-namespace starting
|
114
|
+
# in redis-namespace 2.0
|
115
|
+
if conn.respond_to?(:namespace)
|
116
|
+
conn.redis.info
|
117
|
+
else
|
118
|
+
conn.info
|
121
119
|
end
|
120
|
+
rescue Redis::CommandError => ex
|
121
|
+
# 2850 return fake version when INFO command has (probably) been renamed
|
122
|
+
raise unless /unknown command/.match?(ex.message)
|
123
|
+
FAKE_INFO
|
122
124
|
end
|
123
125
|
end
|
124
126
|
|
@@ -152,18 +154,13 @@ module Sidekiq
|
|
152
154
|
|
153
155
|
def self.default_worker_options=(hash)
|
154
156
|
# stringify
|
155
|
-
@default_worker_options = default_worker_options.merge(Hash[hash.map{|k, v| [k.to_s, v]}])
|
157
|
+
@default_worker_options = default_worker_options.merge(Hash[hash.map { |k, v| [k.to_s, v] }])
|
156
158
|
end
|
159
|
+
|
157
160
|
def self.default_worker_options
|
158
161
|
defined?(@default_worker_options) ? @default_worker_options : DEFAULT_WORKER_OPTIONS
|
159
162
|
end
|
160
163
|
|
161
|
-
def self.default_retries_exhausted=(prok)
|
162
|
-
logger.info { "default_retries_exhausted is deprecated, please use `config.death_handlers << -> {|job, ex| }`" }
|
163
|
-
return nil unless prok
|
164
|
-
death_handlers << prok
|
165
|
-
end
|
166
|
-
|
167
164
|
##
|
168
165
|
# Death handlers are called when all retries for a job have been exhausted and
|
169
166
|
# the job dies. It's the notification to your application
|
@@ -180,15 +177,37 @@ module Sidekiq
|
|
180
177
|
def self.load_json(string)
|
181
178
|
JSON.parse(string)
|
182
179
|
end
|
180
|
+
|
183
181
|
def self.dump_json(object)
|
184
182
|
JSON.generate(object)
|
185
183
|
end
|
186
184
|
|
185
|
+
def self.log_formatter
|
186
|
+
@log_formatter ||= if ENV["DYNO"]
|
187
|
+
Sidekiq::Logger::Formatters::WithoutTimestamp.new
|
188
|
+
else
|
189
|
+
Sidekiq::Logger::Formatters::Pretty.new
|
190
|
+
end
|
191
|
+
end
|
192
|
+
|
193
|
+
def self.log_formatter=(log_formatter)
|
194
|
+
@log_formatter = log_formatter
|
195
|
+
logger.formatter = log_formatter
|
196
|
+
end
|
197
|
+
|
187
198
|
def self.logger
|
188
|
-
Sidekiq::
|
199
|
+
@logger ||= Sidekiq::Logger.new(STDOUT, level: Logger::INFO)
|
189
200
|
end
|
190
|
-
|
191
|
-
|
201
|
+
|
202
|
+
def self.logger=(logger)
|
203
|
+
if logger.nil?
|
204
|
+
self.logger.level = Logger::FATAL
|
205
|
+
return self.logger
|
206
|
+
end
|
207
|
+
|
208
|
+
logger.extend(Sidekiq::LoggingUtils)
|
209
|
+
|
210
|
+
@logger = logger
|
192
211
|
end
|
193
212
|
|
194
213
|
# How frequently Redis should be checked by a random Sidekiq process for
|
@@ -197,7 +216,7 @@ module Sidekiq
|
|
197
216
|
#
|
198
217
|
# See sidekiq/scheduled.rb for an in-depth explanation of this value
|
199
218
|
def self.average_scheduled_poll_interval=(interval)
|
200
|
-
|
219
|
+
options[:average_scheduled_poll_interval] = interval
|
201
220
|
end
|
202
221
|
|
203
222
|
# Register a proc to handle any error which occurs within the Sidekiq process.
|
@@ -208,7 +227,7 @@ module Sidekiq
|
|
208
227
|
#
|
209
228
|
# The default error handler logs errors to Sidekiq.logger.
|
210
229
|
def self.error_handlers
|
211
|
-
|
230
|
+
options[:error_handlers]
|
212
231
|
end
|
213
232
|
|
214
233
|
# Register a block to run at a point in the Sidekiq lifecycle.
|
@@ -234,4 +253,4 @@ module Sidekiq
|
|
234
253
|
class Shutdown < Interrupt; end
|
235
254
|
end
|
236
255
|
|
237
|
-
require
|
256
|
+
require "sidekiq/rails" if defined?(::Rails::Engine)
|
data/lib/sidekiq/api.rb
CHANGED
@@ -1,24 +1,12 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
|
-
require 'sidekiq'
|
3
2
|
|
4
|
-
|
3
|
+
require "sidekiq"
|
5
4
|
|
6
|
-
|
7
|
-
|
8
|
-
cursor = '0'
|
9
|
-
result = []
|
10
|
-
loop do
|
11
|
-
cursor, values = conn.sscan(key, cursor)
|
12
|
-
result.push(*values)
|
13
|
-
break if cursor == '0'
|
14
|
-
end
|
15
|
-
result
|
16
|
-
end
|
17
|
-
end
|
5
|
+
require "zlib"
|
6
|
+
require "base64"
|
18
7
|
|
8
|
+
module Sidekiq
|
19
9
|
class Stats
|
20
|
-
include RedisScanner
|
21
|
-
|
22
10
|
def initialize
|
23
11
|
fetch_stats!
|
24
12
|
end
|
@@ -64,61 +52,65 @@ module Sidekiq
|
|
64
52
|
end
|
65
53
|
|
66
54
|
def fetch_stats!
|
67
|
-
pipe1_res = Sidekiq.redis
|
55
|
+
pipe1_res = Sidekiq.redis { |conn|
|
68
56
|
conn.pipelined do
|
69
|
-
conn.get(
|
70
|
-
conn.get(
|
71
|
-
conn.zcard(
|
72
|
-
conn.zcard(
|
73
|
-
conn.zcard(
|
74
|
-
conn.scard(
|
75
|
-
conn.lrange(
|
57
|
+
conn.get("stat:processed")
|
58
|
+
conn.get("stat:failed")
|
59
|
+
conn.zcard("schedule")
|
60
|
+
conn.zcard("retry")
|
61
|
+
conn.zcard("dead")
|
62
|
+
conn.scard("processes")
|
63
|
+
conn.lrange("queue:default", -1, -1)
|
76
64
|
end
|
77
|
-
|
65
|
+
}
|
78
66
|
|
79
|
-
processes = Sidekiq.redis
|
80
|
-
|
81
|
-
|
67
|
+
processes = Sidekiq.redis { |conn|
|
68
|
+
conn.sscan_each("processes").to_a
|
69
|
+
}
|
82
70
|
|
83
|
-
queues = Sidekiq.redis
|
84
|
-
|
85
|
-
|
71
|
+
queues = Sidekiq.redis { |conn|
|
72
|
+
conn.sscan_each("queues").to_a
|
73
|
+
}
|
86
74
|
|
87
|
-
pipe2_res = Sidekiq.redis
|
75
|
+
pipe2_res = Sidekiq.redis { |conn|
|
88
76
|
conn.pipelined do
|
89
|
-
processes.each {|key| conn.hget(key,
|
90
|
-
queues.each {|queue| conn.llen("queue:#{queue}") }
|
77
|
+
processes.each { |key| conn.hget(key, "busy") }
|
78
|
+
queues.each { |queue| conn.llen("queue:#{queue}") }
|
91
79
|
end
|
92
|
-
|
80
|
+
}
|
93
81
|
|
94
82
|
s = processes.size
|
95
83
|
workers_size = pipe2_res[0...s].map(&:to_i).inject(0, &:+)
|
96
|
-
enqueued
|
84
|
+
enqueued = pipe2_res[s..-1].map(&:to_i).inject(0, &:+)
|
97
85
|
|
98
86
|
default_queue_latency = if (entry = pipe1_res[6].first)
|
99
|
-
|
100
|
-
|
101
|
-
|
102
|
-
|
103
|
-
|
104
|
-
|
105
|
-
|
87
|
+
job = begin
|
88
|
+
Sidekiq.load_json(entry)
|
89
|
+
rescue
|
90
|
+
{}
|
91
|
+
end
|
92
|
+
now = Time.now.to_f
|
93
|
+
thence = job["enqueued_at"] || now
|
94
|
+
now - thence
|
95
|
+
else
|
96
|
+
0
|
97
|
+
end
|
106
98
|
@stats = {
|
107
|
-
processed:
|
108
|
-
failed:
|
109
|
-
scheduled_size:
|
110
|
-
retry_size:
|
111
|
-
dead_size:
|
112
|
-
processes_size:
|
99
|
+
processed: pipe1_res[0].to_i,
|
100
|
+
failed: pipe1_res[1].to_i,
|
101
|
+
scheduled_size: pipe1_res[2],
|
102
|
+
retry_size: pipe1_res[3],
|
103
|
+
dead_size: pipe1_res[4],
|
104
|
+
processes_size: pipe1_res[5],
|
113
105
|
|
114
106
|
default_queue_latency: default_queue_latency,
|
115
|
-
workers_size:
|
116
|
-
enqueued:
|
107
|
+
workers_size: workers_size,
|
108
|
+
enqueued: enqueued,
|
117
109
|
}
|
118
110
|
end
|
119
111
|
|
120
112
|
def reset(*stats)
|
121
|
-
all
|
113
|
+
all = %w[failed processed]
|
122
114
|
stats = stats.empty? ? all : all & stats.flatten.compact.map(&:to_s)
|
123
115
|
|
124
116
|
mset_args = []
|
@@ -138,24 +130,21 @@ module Sidekiq
|
|
138
130
|
end
|
139
131
|
|
140
132
|
class Queues
|
141
|
-
include RedisScanner
|
142
|
-
|
143
133
|
def lengths
|
144
134
|
Sidekiq.redis do |conn|
|
145
|
-
queues =
|
135
|
+
queues = conn.sscan_each("queues").to_a
|
146
136
|
|
147
|
-
lengths = conn.pipelined
|
137
|
+
lengths = conn.pipelined {
|
148
138
|
queues.each do |queue|
|
149
139
|
conn.llen("queue:#{queue}")
|
150
140
|
end
|
151
|
-
|
141
|
+
}
|
152
142
|
|
153
143
|
i = 0
|
154
|
-
array_of_arrays = queues.
|
144
|
+
array_of_arrays = queues.each_with_object({}) { |queue, memo|
|
155
145
|
memo[queue] = lengths[i]
|
156
146
|
i += 1
|
157
|
-
|
158
|
-
end.sort_by { |_, size| size }
|
147
|
+
}.sort_by { |_, size| size }
|
159
148
|
|
160
149
|
Hash[array_of_arrays.reverse]
|
161
150
|
end
|
@@ -222,18 +211,17 @@ module Sidekiq
|
|
222
211
|
#
|
223
212
|
class Queue
|
224
213
|
include Enumerable
|
225
|
-
extend RedisScanner
|
226
214
|
|
227
215
|
##
|
228
216
|
# Return all known queues within Redis.
|
229
217
|
#
|
230
218
|
def self.all
|
231
|
-
Sidekiq.redis { |c|
|
219
|
+
Sidekiq.redis { |c| c.sscan_each("queues").to_a }.sort.map { |q| Sidekiq::Queue.new(q) }
|
232
220
|
end
|
233
221
|
|
234
222
|
attr_reader :name
|
235
223
|
|
236
|
-
def initialize(name="default")
|
224
|
+
def initialize(name = "default")
|
237
225
|
@name = name.to_s
|
238
226
|
@rname = "queue:#{name}"
|
239
227
|
end
|
@@ -253,13 +241,13 @@ module Sidekiq
|
|
253
241
|
#
|
254
242
|
# @return Float
|
255
243
|
def latency
|
256
|
-
entry = Sidekiq.redis
|
244
|
+
entry = Sidekiq.redis { |conn|
|
257
245
|
conn.lrange(@rname, -1, -1)
|
258
|
-
|
246
|
+
}.first
|
259
247
|
return 0 unless entry
|
260
248
|
job = Sidekiq.load_json(entry)
|
261
249
|
now = Time.now.to_f
|
262
|
-
thence = job[
|
250
|
+
thence = job["enqueued_at"] || now
|
263
251
|
now - thence
|
264
252
|
end
|
265
253
|
|
@@ -269,12 +257,12 @@ module Sidekiq
|
|
269
257
|
page = 0
|
270
258
|
page_size = 50
|
271
259
|
|
272
|
-
|
260
|
+
loop do
|
273
261
|
range_start = page * page_size - deleted_size
|
274
|
-
range_end
|
275
|
-
entries = Sidekiq.redis
|
262
|
+
range_end = range_start + page_size - 1
|
263
|
+
entries = Sidekiq.redis { |conn|
|
276
264
|
conn.lrange @rname, range_start, range_end
|
277
|
-
|
265
|
+
}
|
278
266
|
break if entries.empty?
|
279
267
|
page += 1
|
280
268
|
entries.each do |entry|
|
@@ -315,11 +303,11 @@ module Sidekiq
|
|
315
303
|
attr_reader :item
|
316
304
|
attr_reader :value
|
317
305
|
|
318
|
-
def initialize(item, queue_name=nil)
|
306
|
+
def initialize(item, queue_name = nil)
|
319
307
|
@args = nil
|
320
308
|
@value = item
|
321
309
|
@item = item.is_a?(Hash) ? item : parse(item)
|
322
|
-
@queue = queue_name || @item[
|
310
|
+
@queue = queue_name || @item["queue"]
|
323
311
|
end
|
324
312
|
|
325
313
|
def parse(item)
|
@@ -334,7 +322,7 @@ module Sidekiq
|
|
334
322
|
end
|
335
323
|
|
336
324
|
def klass
|
337
|
-
self[
|
325
|
+
self["class"]
|
338
326
|
end
|
339
327
|
|
340
328
|
def display_class
|
@@ -345,16 +333,16 @@ module Sidekiq
|
|
345
333
|
"#{target}.#{method}"
|
346
334
|
end
|
347
335
|
when "ActiveJob::QueueAdapters::SidekiqAdapter::JobWrapper"
|
348
|
-
job_class = @item[
|
349
|
-
if
|
336
|
+
job_class = @item["wrapped"] || args[0]
|
337
|
+
if job_class == "ActionMailer::DeliveryJob" || job_class == "ActionMailer::MailDeliveryJob"
|
350
338
|
# MailerClass#mailer_method
|
351
|
-
args[0][
|
339
|
+
args[0]["arguments"][0..1].join("#")
|
352
340
|
else
|
353
|
-
|
341
|
+
job_class
|
354
342
|
end
|
355
343
|
else
|
356
344
|
klass
|
357
|
-
|
345
|
+
end
|
358
346
|
end
|
359
347
|
|
360
348
|
def display_args
|
@@ -365,53 +353,68 @@ module Sidekiq
|
|
365
353
|
arg
|
366
354
|
end
|
367
355
|
when "ActiveJob::QueueAdapters::SidekiqAdapter::JobWrapper"
|
368
|
-
job_args = self[
|
369
|
-
if
|
356
|
+
job_args = self["wrapped"] ? args[0]["arguments"] : []
|
357
|
+
if (self["wrapped"] || args[0]) == "ActionMailer::DeliveryJob"
|
370
358
|
# remove MailerClass, mailer_method and 'deliver_now'
|
371
359
|
job_args.drop(3)
|
360
|
+
elsif (self["wrapped"] || args[0]) == "ActionMailer::MailDeliveryJob"
|
361
|
+
# remove MailerClass, mailer_method and 'deliver_now'
|
362
|
+
job_args.drop(3).first["args"]
|
372
363
|
else
|
373
364
|
job_args
|
374
365
|
end
|
375
366
|
else
|
376
|
-
if self[
|
367
|
+
if self["encrypt"]
|
377
368
|
# no point in showing 150+ bytes of random garbage
|
378
|
-
args[-1] =
|
369
|
+
args[-1] = "[encrypted data]"
|
379
370
|
end
|
380
371
|
args
|
381
|
-
|
372
|
+
end
|
382
373
|
end
|
383
374
|
|
384
375
|
def args
|
385
|
-
@args || @item[
|
376
|
+
@args || @item["args"]
|
386
377
|
end
|
387
378
|
|
388
379
|
def jid
|
389
|
-
self[
|
380
|
+
self["jid"]
|
390
381
|
end
|
391
382
|
|
392
383
|
def enqueued_at
|
393
|
-
self[
|
384
|
+
self["enqueued_at"] ? Time.at(self["enqueued_at"]).utc : nil
|
394
385
|
end
|
395
386
|
|
396
387
|
def created_at
|
397
|
-
Time.at(self[
|
388
|
+
Time.at(self["created_at"] || self["enqueued_at"] || 0).utc
|
398
389
|
end
|
399
390
|
|
400
|
-
def
|
401
|
-
|
391
|
+
def tags
|
392
|
+
self["tags"] || []
|
402
393
|
end
|
403
394
|
|
395
|
+
def error_backtrace
|
396
|
+
# Cache nil values
|
397
|
+
if defined?(@error_backtrace)
|
398
|
+
@error_backtrace
|
399
|
+
else
|
400
|
+
value = self["error_backtrace"]
|
401
|
+
@error_backtrace = value && uncompress_backtrace(value)
|
402
|
+
end
|
403
|
+
end
|
404
|
+
|
405
|
+
attr_reader :queue
|
406
|
+
|
404
407
|
def latency
|
405
408
|
now = Time.now.to_f
|
406
|
-
now - (@item[
|
409
|
+
now - (@item["enqueued_at"] || @item["created_at"] || now)
|
407
410
|
end
|
408
411
|
|
409
412
|
##
|
410
413
|
# Remove this job from the queue.
|
411
414
|
def delete
|
412
|
-
count = Sidekiq.redis
|
415
|
+
count = Sidekiq.redis { |conn|
|
413
416
|
conn.lrem("queue:#{@queue}", 1, @value)
|
414
|
-
|
417
|
+
}
|
415
418
|
count != 0
|
416
419
|
end
|
417
420
|
|
@@ -425,13 +428,22 @@ module Sidekiq
|
|
425
428
|
private
|
426
429
|
|
427
430
|
def safe_load(content, default)
|
428
|
-
|
429
|
-
|
430
|
-
|
431
|
-
|
432
|
-
|
433
|
-
|
434
|
-
|
431
|
+
yield(*YAML.load(content))
|
432
|
+
rescue => ex
|
433
|
+
# #1761 in dev mode, it's possible to have jobs enqueued which haven't been loaded into
|
434
|
+
# memory yet so the YAML can't be loaded.
|
435
|
+
Sidekiq.logger.warn "Unable to load YAML: #{ex.message}" unless Sidekiq.options[:environment] == "development"
|
436
|
+
default
|
437
|
+
end
|
438
|
+
|
439
|
+
def uncompress_backtrace(backtrace)
|
440
|
+
if backtrace.is_a?(Array)
|
441
|
+
# Handle old jobs with previous backtrace format
|
442
|
+
backtrace
|
443
|
+
else
|
444
|
+
decoded = Base64.decode64(backtrace)
|
445
|
+
uncompressed = Zlib::Inflate.inflate(decoded)
|
446
|
+
Marshal.load(uncompressed)
|
435
447
|
end
|
436
448
|
end
|
437
449
|
end
|
@@ -473,7 +485,7 @@ module Sidekiq
|
|
473
485
|
def retry
|
474
486
|
remove_job do |message|
|
475
487
|
msg = Sidekiq.load_json(message)
|
476
|
-
msg[
|
488
|
+
msg["retry_count"] -= 1 if msg["retry_count"]
|
477
489
|
Sidekiq::Client.push(msg)
|
478
490
|
end
|
479
491
|
end
|
@@ -487,31 +499,31 @@ module Sidekiq
|
|
487
499
|
end
|
488
500
|
|
489
501
|
def error?
|
490
|
-
!!item[
|
502
|
+
!!item["error_class"]
|
491
503
|
end
|
492
504
|
|
493
505
|
private
|
494
506
|
|
495
507
|
def remove_job
|
496
508
|
Sidekiq.redis do |conn|
|
497
|
-
results = conn.multi
|
509
|
+
results = conn.multi {
|
498
510
|
conn.zrangebyscore(parent.name, score, score)
|
499
511
|
conn.zremrangebyscore(parent.name, score, score)
|
500
|
-
|
512
|
+
}.first
|
501
513
|
|
502
514
|
if results.size == 1
|
503
515
|
yield results.first
|
504
516
|
else
|
505
517
|
# multiple jobs with the same score
|
506
518
|
# find the one with the right JID and push it
|
507
|
-
hash = results.group_by
|
519
|
+
hash = results.group_by { |message|
|
508
520
|
if message.index(jid)
|
509
521
|
msg = Sidekiq.load_json(message)
|
510
|
-
msg[
|
522
|
+
msg["jid"] == jid
|
511
523
|
else
|
512
524
|
false
|
513
525
|
end
|
514
|
-
|
526
|
+
}
|
515
527
|
|
516
528
|
msg = hash.fetch(true, []).first
|
517
529
|
yield msg if msg
|
@@ -525,7 +537,6 @@ module Sidekiq
|
|
525
537
|
end
|
526
538
|
end
|
527
539
|
end
|
528
|
-
|
529
540
|
end
|
530
541
|
|
531
542
|
class SortedSet
|
@@ -542,6 +553,17 @@ module Sidekiq
|
|
542
553
|
Sidekiq.redis { |c| c.zcard(name) }
|
543
554
|
end
|
544
555
|
|
556
|
+
def scan(match, count = 100)
|
557
|
+
return to_enum(:scan, match) unless block_given?
|
558
|
+
|
559
|
+
match = "*#{match}*" unless match.include?("*")
|
560
|
+
Sidekiq.redis do |conn|
|
561
|
+
conn.zscan_each(name, match: match, count: count) do |entry, score|
|
562
|
+
yield SortedEntry.new(self, score, entry)
|
563
|
+
end
|
564
|
+
end
|
565
|
+
end
|
566
|
+
|
545
567
|
def clear
|
546
568
|
Sidekiq.redis do |conn|
|
547
569
|
conn.del(name)
|
@@ -551,7 +573,6 @@ module Sidekiq
|
|
551
573
|
end
|
552
574
|
|
553
575
|
class JobSet < SortedSet
|
554
|
-
|
555
576
|
def schedule(timestamp, message)
|
556
577
|
Sidekiq.redis do |conn|
|
557
578
|
conn.zadd(name, timestamp.to_f.to_s, Sidekiq.dump_json(message))
|
@@ -564,44 +585,55 @@ module Sidekiq
|
|
564
585
|
page = -1
|
565
586
|
page_size = 50
|
566
587
|
|
567
|
-
|
588
|
+
loop do
|
568
589
|
range_start = page * page_size + offset_size
|
569
|
-
range_end
|
570
|
-
elements = Sidekiq.redis
|
590
|
+
range_end = range_start + page_size - 1
|
591
|
+
elements = Sidekiq.redis { |conn|
|
571
592
|
conn.zrange name, range_start, range_end, with_scores: true
|
572
|
-
|
593
|
+
}
|
573
594
|
break if elements.empty?
|
574
595
|
page -= 1
|
575
|
-
elements.
|
596
|
+
elements.reverse_each do |element, score|
|
576
597
|
yield SortedEntry.new(self, score, element)
|
577
598
|
end
|
578
599
|
offset_size = initial_size - @_size
|
579
600
|
end
|
580
601
|
end
|
581
602
|
|
603
|
+
##
|
604
|
+
# Fetch jobs that match a given time or Range. Job ID is an
|
605
|
+
# optional second argument.
|
582
606
|
def fetch(score, jid = nil)
|
583
|
-
|
584
|
-
|
585
|
-
|
586
|
-
|
587
|
-
elements.inject([]) do |result, element|
|
588
|
-
entry = SortedEntry.new(self, score, element)
|
589
|
-
if jid
|
590
|
-
result << entry if entry.jid == jid
|
607
|
+
begin_score, end_score =
|
608
|
+
if score.is_a?(Range)
|
609
|
+
[score.first, score.last]
|
591
610
|
else
|
592
|
-
|
611
|
+
[score, score]
|
593
612
|
end
|
594
|
-
|
613
|
+
|
614
|
+
elements = Sidekiq.redis { |conn|
|
615
|
+
conn.zrangebyscore(name, begin_score, end_score, with_scores: true)
|
616
|
+
}
|
617
|
+
|
618
|
+
elements.each_with_object([]) do |element, result|
|
619
|
+
data, job_score = element
|
620
|
+
entry = SortedEntry.new(self, job_score, data)
|
621
|
+
result << entry if jid.nil? || entry.jid == jid
|
595
622
|
end
|
596
623
|
end
|
597
624
|
|
598
625
|
##
|
599
626
|
# Find the job with the given JID within this sorted set.
|
600
|
-
#
|
601
|
-
# This is a slow, inefficient operation. Do not use under
|
602
|
-
# normal conditions. Sidekiq Pro contains a faster version.
|
627
|
+
# This is a slower O(n) operation. Do not use for app logic.
|
603
628
|
def find_job(jid)
|
604
|
-
|
629
|
+
Sidekiq.redis do |conn|
|
630
|
+
conn.zscan_each(name, match: "*#{jid}*", count: 100) do |entry, score|
|
631
|
+
job = JSON.parse(entry)
|
632
|
+
matched = job["jid"] == jid
|
633
|
+
return SortedEntry.new(self, score, entry) if matched
|
634
|
+
end
|
635
|
+
end
|
636
|
+
nil
|
605
637
|
end
|
606
638
|
|
607
639
|
def delete_by_value(name, value)
|
@@ -622,7 +654,6 @@ module Sidekiq
|
|
622
654
|
@_size -= 1 if ret
|
623
655
|
break ret
|
624
656
|
end
|
625
|
-
false
|
626
657
|
end
|
627
658
|
end
|
628
659
|
end
|
@@ -644,7 +675,7 @@ module Sidekiq
|
|
644
675
|
# end.map(&:delete)
|
645
676
|
class ScheduledSet < JobSet
|
646
677
|
def initialize
|
647
|
-
super
|
678
|
+
super "schedule"
|
648
679
|
end
|
649
680
|
end
|
650
681
|
|
@@ -662,19 +693,15 @@ module Sidekiq
|
|
662
693
|
# end.map(&:delete)
|
663
694
|
class RetrySet < JobSet
|
664
695
|
def initialize
|
665
|
-
super
|
696
|
+
super "retry"
|
666
697
|
end
|
667
698
|
|
668
699
|
def retry_all
|
669
|
-
while size > 0
|
670
|
-
each(&:retry)
|
671
|
-
end
|
700
|
+
each(&:retry) while size > 0
|
672
701
|
end
|
673
702
|
|
674
703
|
def kill_all
|
675
|
-
while size > 0
|
676
|
-
each(&:kill)
|
677
|
-
end
|
704
|
+
each(&:kill) while size > 0
|
678
705
|
end
|
679
706
|
end
|
680
707
|
|
@@ -683,15 +710,15 @@ module Sidekiq
|
|
683
710
|
#
|
684
711
|
class DeadSet < JobSet
|
685
712
|
def initialize
|
686
|
-
super
|
713
|
+
super "dead"
|
687
714
|
end
|
688
715
|
|
689
|
-
def kill(message, opts={})
|
716
|
+
def kill(message, opts = {})
|
690
717
|
now = Time.now.to_f
|
691
718
|
Sidekiq.redis do |conn|
|
692
719
|
conn.multi do
|
693
720
|
conn.zadd(name, now.to_s, message)
|
694
|
-
conn.zremrangebyscore(name,
|
721
|
+
conn.zremrangebyscore(name, "-inf", now - self.class.timeout)
|
695
722
|
conn.zremrangebyrank(name, 0, - self.class.max_jobs)
|
696
723
|
end
|
697
724
|
end
|
@@ -708,9 +735,7 @@ module Sidekiq
|
|
708
735
|
end
|
709
736
|
|
710
737
|
def retry_all
|
711
|
-
while size > 0
|
712
|
-
each(&:retry)
|
713
|
-
end
|
738
|
+
each(&:retry) while size > 0
|
714
739
|
end
|
715
740
|
|
716
741
|
def self.max_jobs
|
@@ -724,16 +749,15 @@ module Sidekiq
|
|
724
749
|
|
725
750
|
##
|
726
751
|
# Enumerates the set of Sidekiq processes which are actively working
|
727
|
-
# right now. Each process
|
752
|
+
# right now. Each process sends a heartbeat to Redis every 5 seconds
|
728
753
|
# so this set should be relatively accurate, barring network partitions.
|
729
754
|
#
|
730
755
|
# Yields a Sidekiq::Process.
|
731
756
|
#
|
732
757
|
class ProcessSet
|
733
758
|
include Enumerable
|
734
|
-
include RedisScanner
|
735
759
|
|
736
|
-
def initialize(clean_plz=true)
|
760
|
+
def initialize(clean_plz = true)
|
737
761
|
cleanup if clean_plz
|
738
762
|
end
|
739
763
|
|
@@ -742,12 +766,12 @@ module Sidekiq
|
|
742
766
|
def cleanup
|
743
767
|
count = 0
|
744
768
|
Sidekiq.redis do |conn|
|
745
|
-
procs =
|
746
|
-
heartbeats = conn.pipelined
|
769
|
+
procs = conn.sscan_each("processes").to_a.sort
|
770
|
+
heartbeats = conn.pipelined {
|
747
771
|
procs.each do |key|
|
748
|
-
conn.hget(key,
|
772
|
+
conn.hget(key, "info")
|
749
773
|
end
|
750
|
-
|
774
|
+
}
|
751
775
|
|
752
776
|
# the hash named key has an expiry of 60 seconds.
|
753
777
|
# if it's not found, that means the process has not reported
|
@@ -756,23 +780,23 @@ module Sidekiq
|
|
756
780
|
heartbeats.each_with_index do |beat, i|
|
757
781
|
to_prune << procs[i] if beat.nil?
|
758
782
|
end
|
759
|
-
count = conn.srem(
|
783
|
+
count = conn.srem("processes", to_prune) unless to_prune.empty?
|
760
784
|
end
|
761
785
|
count
|
762
786
|
end
|
763
787
|
|
764
788
|
def each
|
765
|
-
procs = Sidekiq.redis { |conn|
|
789
|
+
procs = Sidekiq.redis { |conn| conn.sscan_each("processes").to_a }.sort
|
766
790
|
|
767
791
|
Sidekiq.redis do |conn|
|
768
792
|
# We're making a tradeoff here between consuming more memory instead of
|
769
793
|
# making more roundtrips to Redis, but if you have hundreds or thousands of workers,
|
770
794
|
# you'll be happier this way
|
771
|
-
result = conn.pipelined
|
795
|
+
result = conn.pipelined {
|
772
796
|
procs.each do |key|
|
773
|
-
conn.hmget(key,
|
797
|
+
conn.hmget(key, "info", "busy", "beat", "quiet")
|
774
798
|
end
|
775
|
-
|
799
|
+
}
|
776
800
|
|
777
801
|
result.each do |info, busy, at_s, quiet|
|
778
802
|
# If a process is stopped between when we query Redis for `procs` and
|
@@ -781,7 +805,7 @@ module Sidekiq
|
|
781
805
|
next if info.nil?
|
782
806
|
|
783
807
|
hash = Sidekiq.load_json(info)
|
784
|
-
yield Process.new(hash.merge(
|
808
|
+
yield Process.new(hash.merge("busy" => busy.to_i, "beat" => at_s.to_f, "quiet" => quiet))
|
785
809
|
end
|
786
810
|
end
|
787
811
|
|
@@ -793,7 +817,7 @@ module Sidekiq
|
|
793
817
|
# contains Sidekiq processes which have sent a heartbeat within the last
|
794
818
|
# 60 seconds.
|
795
819
|
def size
|
796
|
-
Sidekiq.redis { |conn| conn.scard(
|
820
|
+
Sidekiq.redis { |conn| conn.scard("processes") }
|
797
821
|
end
|
798
822
|
|
799
823
|
# Returns the identity of the current cluster leader or "" if no leader.
|
@@ -801,9 +825,9 @@ module Sidekiq
|
|
801
825
|
# or Sidekiq Pro.
|
802
826
|
def leader
|
803
827
|
@leader ||= begin
|
804
|
-
x = Sidekiq.redis {|c| c.get("dear-leader") }
|
828
|
+
x = Sidekiq.redis { |c| c.get("dear-leader") }
|
805
829
|
# need a non-falsy value so we can memoize
|
806
|
-
x
|
830
|
+
x ||= ""
|
807
831
|
x
|
808
832
|
end
|
809
833
|
end
|
@@ -830,11 +854,11 @@ module Sidekiq
|
|
830
854
|
end
|
831
855
|
|
832
856
|
def tag
|
833
|
-
self[
|
857
|
+
self["tag"]
|
834
858
|
end
|
835
859
|
|
836
860
|
def labels
|
837
|
-
Array(self[
|
861
|
+
Array(self["labels"])
|
838
862
|
end
|
839
863
|
|
840
864
|
def [](key)
|
@@ -842,23 +866,23 @@ module Sidekiq
|
|
842
866
|
end
|
843
867
|
|
844
868
|
def identity
|
845
|
-
self[
|
869
|
+
self["identity"]
|
846
870
|
end
|
847
871
|
|
848
872
|
def quiet!
|
849
|
-
signal(
|
873
|
+
signal("TSTP")
|
850
874
|
end
|
851
875
|
|
852
876
|
def stop!
|
853
|
-
signal(
|
877
|
+
signal("TERM")
|
854
878
|
end
|
855
879
|
|
856
880
|
def dump_threads
|
857
|
-
signal(
|
881
|
+
signal("TTIN")
|
858
882
|
end
|
859
883
|
|
860
884
|
def stopping?
|
861
|
-
self[
|
885
|
+
self["quiet"] == "true"
|
862
886
|
end
|
863
887
|
|
864
888
|
private
|
@@ -872,7 +896,6 @@ module Sidekiq
|
|
872
896
|
end
|
873
897
|
end
|
874
898
|
end
|
875
|
-
|
876
899
|
end
|
877
900
|
|
878
901
|
##
|
@@ -897,16 +920,15 @@ module Sidekiq
|
|
897
920
|
#
|
898
921
|
class Workers
|
899
922
|
include Enumerable
|
900
|
-
include RedisScanner
|
901
923
|
|
902
924
|
def each
|
903
925
|
Sidekiq.redis do |conn|
|
904
|
-
procs =
|
926
|
+
procs = conn.sscan_each("processes").to_a
|
905
927
|
procs.sort.each do |key|
|
906
|
-
valid, workers = conn.pipelined
|
928
|
+
valid, workers = conn.pipelined {
|
907
929
|
conn.exists(key)
|
908
930
|
conn.hgetall("#{key}:workers")
|
909
|
-
|
931
|
+
}
|
910
932
|
next unless valid
|
911
933
|
workers.each_pair do |tid, json|
|
912
934
|
yield key, tid, Sidekiq.load_json(json)
|
@@ -923,18 +945,17 @@ module Sidekiq
|
|
923
945
|
# which can easily get out of sync with crashy processes.
|
924
946
|
def size
|
925
947
|
Sidekiq.redis do |conn|
|
926
|
-
procs =
|
948
|
+
procs = conn.sscan_each("processes").to_a
|
927
949
|
if procs.empty?
|
928
950
|
0
|
929
951
|
else
|
930
|
-
conn.pipelined
|
952
|
+
conn.pipelined {
|
931
953
|
procs.each do |key|
|
932
|
-
conn.hget(key,
|
954
|
+
conn.hget(key, "busy")
|
933
955
|
end
|
934
|
-
|
956
|
+
}.map(&:to_i).inject(:+)
|
935
957
|
end
|
936
958
|
end
|
937
959
|
end
|
938
960
|
end
|
939
|
-
|
940
961
|
end
|