sidekiq 6.2.1 → 6.3.1
Sign up to get free protection for your applications and to get access to all the features.
Potentially problematic release.
This version of sidekiq might be problematic. Click here for more details.
- checksums.yaml +4 -4
- data/Changes.md +52 -1
- data/LICENSE +1 -1
- data/README.md +2 -2
- data/lib/sidekiq/api.rb +80 -56
- data/lib/sidekiq/cli.rb +10 -2
- data/lib/sidekiq/client.rb +2 -2
- data/lib/sidekiq/extensions/generic_proxy.rb +3 -1
- data/lib/sidekiq/fetch.rb +5 -4
- data/lib/sidekiq/job.rb +13 -0
- data/lib/sidekiq/job_logger.rb +1 -1
- data/lib/sidekiq/job_retry.rb +3 -7
- data/lib/sidekiq/launcher.rb +18 -18
- data/lib/sidekiq/middleware/chain.rb +5 -3
- data/lib/sidekiq/middleware/current_attributes.rb +52 -0
- data/lib/sidekiq/rails.rb +11 -0
- data/lib/sidekiq/redis_connection.rb +4 -6
- data/lib/sidekiq/scheduled.rb +40 -15
- data/lib/sidekiq/testing.rb +1 -3
- data/lib/sidekiq/version.rb +1 -1
- data/lib/sidekiq/web/action.rb +1 -1
- data/lib/sidekiq/web/application.rb +4 -4
- data/lib/sidekiq/web/helpers.rb +9 -21
- data/lib/sidekiq/web.rb +4 -3
- data/lib/sidekiq/worker.rb +72 -5
- data/lib/sidekiq.rb +3 -1
- data/sidekiq.gemspec +1 -1
- data/web/assets/javascripts/application.js +82 -61
- data/web/assets/javascripts/dashboard.js +51 -51
- data/web/assets/stylesheets/application-dark.css +18 -31
- data/web/assets/stylesheets/application-rtl.css +0 -4
- data/web/assets/stylesheets/application.css +21 -233
- data/web/locales/ar.yml +8 -2
- data/web/locales/en.yml +4 -1
- data/web/locales/es.yml +18 -2
- data/web/locales/fr.yml +7 -0
- data/web/locales/ja.yml +3 -0
- data/web/locales/lt.yml +1 -1
- data/web/views/_footer.erb +1 -1
- data/web/views/_job_info.erb +1 -1
- data/web/views/_poll_link.erb +2 -5
- data/web/views/_summary.erb +7 -7
- data/web/views/busy.erb +5 -5
- data/web/views/dashboard.erb +22 -14
- data/web/views/dead.erb +1 -1
- data/web/views/layout.erb +1 -1
- data/web/views/morgue.erb +6 -6
- data/web/views/queue.erb +10 -10
- data/web/views/queues.erb +3 -3
- data/web/views/retries.erb +7 -7
- data/web/views/retry.erb +1 -1
- data/web/views/scheduled.erb +1 -1
- metadata +5 -3
data/lib/sidekiq/scheduled.rb
CHANGED
@@ -9,29 +9,48 @@ module Sidekiq
|
|
9
9
|
SETS = %w[retry schedule]
|
10
10
|
|
11
11
|
class Enq
|
12
|
+
LUA_ZPOPBYSCORE = <<~LUA
|
13
|
+
local key, now = KEYS[1], ARGV[1]
|
14
|
+
local jobs = redis.call("zrangebyscore", key, "-inf", now, "limit", 0, 1)
|
15
|
+
if jobs[1] then
|
16
|
+
redis.call("zrem", key, jobs[1])
|
17
|
+
return jobs[1]
|
18
|
+
end
|
19
|
+
LUA
|
20
|
+
|
21
|
+
def initialize
|
22
|
+
@lua_zpopbyscore_sha = nil
|
23
|
+
end
|
24
|
+
|
12
25
|
def enqueue_jobs(now = Time.now.to_f.to_s, sorted_sets = SETS)
|
13
26
|
# A job's "score" in Redis is the time at which it should be processed.
|
14
27
|
# Just check Redis for the set of jobs with a timestamp before now.
|
15
28
|
Sidekiq.redis do |conn|
|
16
29
|
sorted_sets.each do |sorted_set|
|
17
|
-
# Get next
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
# the queue, it's because another process already popped it so we can move on to the
|
25
|
-
# next one.
|
26
|
-
if conn.zrem(sorted_set, job)
|
27
|
-
Sidekiq::Client.push(Sidekiq.load_json(job))
|
28
|
-
Sidekiq.logger.debug { "enqueued #{sorted_set}: #{job}" }
|
29
|
-
end
|
30
|
-
end
|
30
|
+
# Get next item in the queue with score (time to execute) <= now.
|
31
|
+
# We need to go through the list one at a time to reduce the risk of something
|
32
|
+
# going wrong between the time jobs are popped from the scheduled queue and when
|
33
|
+
# they are pushed onto a work queue and losing the jobs.
|
34
|
+
while (job = zpopbyscore(conn, keys: [sorted_set], argv: [now]))
|
35
|
+
Sidekiq::Client.push(Sidekiq.load_json(job))
|
36
|
+
Sidekiq.logger.debug { "enqueued #{sorted_set}: #{job}" }
|
31
37
|
end
|
32
38
|
end
|
33
39
|
end
|
34
40
|
end
|
41
|
+
|
42
|
+
private
|
43
|
+
|
44
|
+
def zpopbyscore(conn, keys: nil, argv: nil)
|
45
|
+
@lua_zpopbyscore_sha = conn.script(:load, LUA_ZPOPBYSCORE) if @lua_zpopbyscore_sha.nil?
|
46
|
+
|
47
|
+
conn.evalsha(@lua_zpopbyscore_sha, keys: keys, argv: argv)
|
48
|
+
rescue Redis::CommandError => e
|
49
|
+
raise unless e.message.start_with?("NOSCRIPT")
|
50
|
+
|
51
|
+
@lua_zpopbyscore_sha = nil
|
52
|
+
retry
|
53
|
+
end
|
35
54
|
end
|
36
55
|
|
37
56
|
##
|
@@ -49,6 +68,7 @@ module Sidekiq
|
|
49
68
|
@sleeper = ConnectionPool::TimedStack.new
|
50
69
|
@done = false
|
51
70
|
@thread = nil
|
71
|
+
@count_calls = 0
|
52
72
|
end
|
53
73
|
|
54
74
|
# Shut down this instance, will pause until the thread is dead.
|
@@ -152,8 +172,13 @@ module Sidekiq
|
|
152
172
|
end
|
153
173
|
|
154
174
|
def process_count
|
155
|
-
|
175
|
+
# The work buried within Sidekiq::ProcessSet#cleanup can be
|
176
|
+
# expensive at scale. Cut it down by 90% with this counter.
|
177
|
+
# NB: This method is only called by the scheduler thread so we
|
178
|
+
# don't need to worry about the thread safety of +=.
|
179
|
+
pcount = Sidekiq::ProcessSet.new(@count_calls % 10 == 0).size
|
156
180
|
pcount = 1 if pcount == 0
|
181
|
+
@count_calls += 1
|
157
182
|
pcount
|
158
183
|
end
|
159
184
|
|
data/lib/sidekiq/testing.rb
CHANGED
@@ -338,7 +338,5 @@ module Sidekiq
|
|
338
338
|
end
|
339
339
|
|
340
340
|
if defined?(::Rails) && Rails.respond_to?(:env) && !Rails.env.test? && !$TESTING
|
341
|
-
|
342
|
-
puts("⛔️ WARNING: Sidekiq testing API enabled, but this is not the test environment. Your jobs will not go to Redis.")
|
343
|
-
puts("**************************************************")
|
341
|
+
warn("⛔️ WARNING: Sidekiq testing API enabled, but this is not the test environment. Your jobs will not go to Redis.", uplevel: 1)
|
344
342
|
end
|
data/lib/sidekiq/version.rb
CHANGED
data/lib/sidekiq/web/action.rb
CHANGED
@@ -68,7 +68,7 @@ module Sidekiq
|
|
68
68
|
end
|
69
69
|
|
70
70
|
def json(payload)
|
71
|
-
[200, {"Content-Type" => "application/json", "Cache-Control" => "no-
|
71
|
+
[200, {"Content-Type" => "application/json", "Cache-Control" => "private, no-store"}, [Sidekiq.dump_json(payload)]]
|
72
72
|
end
|
73
73
|
|
74
74
|
def initialize(env, block)
|
@@ -91,8 +91,8 @@ module Sidekiq
|
|
91
91
|
|
92
92
|
@count = (params["count"] || 25).to_i
|
93
93
|
@queue = Sidekiq::Queue.new(@name)
|
94
|
-
(@current_page, @total_size, @
|
95
|
-
@
|
94
|
+
(@current_page, @total_size, @jobs) = page("queue:#{@name}", params["page"], @count, reverse: params["direction"] == "asc")
|
95
|
+
@jobs = @jobs.map { |msg| Sidekiq::JobRecord.new(msg, @name) }
|
96
96
|
|
97
97
|
erb(:queue)
|
98
98
|
end
|
@@ -113,7 +113,7 @@ module Sidekiq
|
|
113
113
|
|
114
114
|
post "/queues/:name/delete" do
|
115
115
|
name = route_params[:name]
|
116
|
-
Sidekiq::
|
116
|
+
Sidekiq::JobRecord.new(params["key_val"], name).delete
|
117
117
|
|
118
118
|
redirect_with_query("#{root_path}queues/#{CGI.escape(name)}")
|
119
119
|
end
|
@@ -314,7 +314,7 @@ module Sidekiq
|
|
314
314
|
# rendered content goes here
|
315
315
|
headers = {
|
316
316
|
"Content-Type" => "text/html",
|
317
|
-
"Cache-Control" => "no-
|
317
|
+
"Cache-Control" => "private, no-store",
|
318
318
|
"Content-Language" => action.locale,
|
319
319
|
"Content-Security-Policy" => CSP_HEADER
|
320
320
|
}
|
data/lib/sidekiq/web/helpers.rb
CHANGED
@@ -10,14 +10,13 @@ module Sidekiq
|
|
10
10
|
module WebHelpers
|
11
11
|
def strings(lang)
|
12
12
|
@strings ||= {}
|
13
|
-
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
end
|
13
|
+
|
14
|
+
# Allow sidekiq-web extensions to add locale paths
|
15
|
+
# so extensions can be localized
|
16
|
+
@strings[lang] ||= settings.locales.each_with_object({}) do |path, global|
|
17
|
+
find_locale_files(lang).each do |file|
|
18
|
+
strs = YAML.load(File.open(file))
|
19
|
+
global.merge!(strs[lang])
|
21
20
|
end
|
22
21
|
end
|
23
22
|
end
|
@@ -71,17 +70,6 @@ module Sidekiq
|
|
71
70
|
@head_html.join if defined?(@head_html)
|
72
71
|
end
|
73
72
|
|
74
|
-
def poll_path
|
75
|
-
if current_path != "" && params["poll"]
|
76
|
-
path = root_path + current_path
|
77
|
-
query_string = to_query_string(params.slice(*params.keys - %w[page poll]))
|
78
|
-
path += "?#{query_string}" unless query_string.empty?
|
79
|
-
path
|
80
|
-
else
|
81
|
-
""
|
82
|
-
end
|
83
|
-
end
|
84
|
-
|
85
73
|
def text_direction
|
86
74
|
get_locale["TextDirection"] || "ltr"
|
87
75
|
end
|
@@ -126,7 +114,7 @@ module Sidekiq
|
|
126
114
|
# within is used by Sidekiq Pro
|
127
115
|
def display_tags(job, within = nil)
|
128
116
|
job.tags.map { |tag|
|
129
|
-
"<span class='
|
117
|
+
"<span class='label label-info jobtag'>#{::Rack::Utils.escape_html(tag)}</span>"
|
130
118
|
}.join(" ")
|
131
119
|
end
|
132
120
|
|
@@ -204,7 +192,7 @@ module Sidekiq
|
|
204
192
|
[score.to_f, jid]
|
205
193
|
end
|
206
194
|
|
207
|
-
SAFE_QPARAMS = %w[page
|
195
|
+
SAFE_QPARAMS = %w[page direction]
|
208
196
|
|
209
197
|
# Merge options with current params, filter safe params, and stringify to query string
|
210
198
|
def qparams(options)
|
data/lib/sidekiq/web.rb
CHANGED
@@ -143,13 +143,14 @@ module Sidekiq
|
|
143
143
|
klass = self.class
|
144
144
|
m = middlewares
|
145
145
|
|
146
|
+
rules = []
|
147
|
+
rules = [[:all, {"Cache-Control" => "public, max-age=86400"}]] unless ENV["SIDEKIQ_WEB_TESTING"]
|
148
|
+
|
146
149
|
::Rack::Builder.new do
|
147
150
|
use Rack::Static, urls: ["/stylesheets", "/images", "/javascripts"],
|
148
151
|
root: ASSETS,
|
149
152
|
cascade: true,
|
150
|
-
header_rules:
|
151
|
-
[:all, {"Cache-Control" => "public, max-age=86400"}]
|
152
|
-
]
|
153
|
+
header_rules: rules
|
153
154
|
m.each { |middleware, block| use(*middleware, &block) }
|
154
155
|
use Sidekiq::Web::CsrfProtection unless $TESTING
|
155
156
|
run WebApplication.new(klass)
|
data/lib/sidekiq/worker.rb
CHANGED
@@ -9,6 +9,7 @@ module Sidekiq
|
|
9
9
|
#
|
10
10
|
# class HardWorker
|
11
11
|
# include Sidekiq::Worker
|
12
|
+
# sidekiq_options queue: 'critical', retry: 5
|
12
13
|
#
|
13
14
|
# def perform(*args)
|
14
15
|
# # do some work
|
@@ -20,6 +21,26 @@ module Sidekiq
|
|
20
21
|
# HardWorker.perform_async(1, 2, 3)
|
21
22
|
#
|
22
23
|
# Note that perform_async is a class method, perform is an instance method.
|
24
|
+
#
|
25
|
+
# Sidekiq::Worker also includes several APIs to provide compatibility with
|
26
|
+
# ActiveJob.
|
27
|
+
#
|
28
|
+
# class SomeWorker
|
29
|
+
# include Sidekiq::Worker
|
30
|
+
# queue_as :critical
|
31
|
+
#
|
32
|
+
# def perform(...)
|
33
|
+
# end
|
34
|
+
# end
|
35
|
+
#
|
36
|
+
# SomeWorker.set(wait_until: 1.hour).perform_async(123)
|
37
|
+
#
|
38
|
+
# Note that arguments passed to the job must still obey Sidekiq's
|
39
|
+
# best practice for simple, JSON-native data types. Sidekiq will not
|
40
|
+
# implement ActiveJob's more complex argument serialization. For
|
41
|
+
# this reason, we don't implement `perform_later` as our call semantics
|
42
|
+
# are very different.
|
43
|
+
#
|
23
44
|
module Worker
|
24
45
|
##
|
25
46
|
# The Options module is extracted so we can include it in ActiveJob::Base
|
@@ -153,10 +174,16 @@ module Sidekiq
|
|
153
174
|
def initialize(klass, opts)
|
154
175
|
@klass = klass
|
155
176
|
@opts = opts
|
177
|
+
|
178
|
+
# ActiveJob compatibility
|
179
|
+
interval = @opts.delete(:wait_until) || @opts.delete(:wait)
|
180
|
+
at(interval) if interval
|
156
181
|
end
|
157
182
|
|
158
183
|
def set(options)
|
184
|
+
interval = options.delete(:wait_until) || options.delete(:wait)
|
159
185
|
@opts.merge!(options)
|
186
|
+
at(interval) if interval
|
160
187
|
self
|
161
188
|
end
|
162
189
|
|
@@ -164,19 +191,29 @@ module Sidekiq
|
|
164
191
|
@klass.client_push(@opts.merge("args" => args, "class" => @klass))
|
165
192
|
end
|
166
193
|
|
194
|
+
def perform_bulk(args, batch_size: 1_000)
|
195
|
+
args.each_slice(batch_size).flat_map do |slice|
|
196
|
+
Sidekiq::Client.push_bulk(@opts.merge("class" => @klass, "args" => slice))
|
197
|
+
end
|
198
|
+
end
|
199
|
+
|
167
200
|
# +interval+ must be a timestamp, numeric or something that acts
|
168
201
|
# numeric (like an activesupport time interval).
|
169
202
|
def perform_in(interval, *args)
|
203
|
+
at(interval).perform_async(*args)
|
204
|
+
end
|
205
|
+
alias_method :perform_at, :perform_in
|
206
|
+
|
207
|
+
private
|
208
|
+
|
209
|
+
def at(interval)
|
170
210
|
int = interval.to_f
|
171
211
|
now = Time.now.to_f
|
172
212
|
ts = (int < 1_000_000_000 ? now + int : int)
|
173
|
-
|
174
|
-
payload = @opts.merge("class" => @klass, "args" => args)
|
175
213
|
# Optimization to enqueue something now that is scheduled to go out now or in the past
|
176
|
-
|
177
|
-
|
214
|
+
@opts["at"] = ts if ts > now
|
215
|
+
self
|
178
216
|
end
|
179
|
-
alias_method :perform_at, :perform_in
|
180
217
|
end
|
181
218
|
|
182
219
|
module ClassMethods
|
@@ -192,6 +229,10 @@ module Sidekiq
|
|
192
229
|
raise ArgumentError, "Do not call .delay_until on a Sidekiq::Worker class, call .perform_at"
|
193
230
|
end
|
194
231
|
|
232
|
+
def queue_as(q)
|
233
|
+
sidekiq_options("queue" => q.to_s)
|
234
|
+
end
|
235
|
+
|
195
236
|
def set(options)
|
196
237
|
Setter.new(self, options)
|
197
238
|
end
|
@@ -200,6 +241,32 @@ module Sidekiq
|
|
200
241
|
client_push("class" => self, "args" => args)
|
201
242
|
end
|
202
243
|
|
244
|
+
##
|
245
|
+
# Push a large number of jobs to Redis, while limiting the batch of
|
246
|
+
# each job payload to 1,000. This method helps cut down on the number
|
247
|
+
# of round trips to Redis, which can increase the performance of enqueueing
|
248
|
+
# large numbers of jobs.
|
249
|
+
#
|
250
|
+
# +items+ must be an Array of Arrays.
|
251
|
+
#
|
252
|
+
# For finer-grained control, use `Sidekiq::Client.push_bulk` directly.
|
253
|
+
#
|
254
|
+
# Example (3 Redis round trips):
|
255
|
+
#
|
256
|
+
# SomeWorker.perform_async(1)
|
257
|
+
# SomeWorker.perform_async(2)
|
258
|
+
# SomeWorker.perform_async(3)
|
259
|
+
#
|
260
|
+
# Would instead become (1 Redis round trip):
|
261
|
+
#
|
262
|
+
# SomeWorker.perform_bulk([[1], [2], [3]])
|
263
|
+
#
|
264
|
+
def perform_bulk(items, batch_size: 1_000)
|
265
|
+
items.each_slice(batch_size).flat_map do |slice|
|
266
|
+
Sidekiq::Client.push_bulk("class" => self, "args" => slice)
|
267
|
+
end
|
268
|
+
end
|
269
|
+
|
203
270
|
# +interval+ must be a timestamp, numeric or something that acts
|
204
271
|
# numeric (like an activesupport time interval).
|
205
272
|
def perform_in(interval, *args)
|
data/lib/sidekiq.rb
CHANGED
@@ -6,6 +6,7 @@ fail "Sidekiq #{Sidekiq::VERSION} does not support Ruby versions below 2.5.0." i
|
|
6
6
|
require "sidekiq/logger"
|
7
7
|
require "sidekiq/client"
|
8
8
|
require "sidekiq/worker"
|
9
|
+
require "sidekiq/job"
|
9
10
|
require "sidekiq/redis_connection"
|
10
11
|
require "sidekiq/delay"
|
11
12
|
|
@@ -100,7 +101,8 @@ module Sidekiq
|
|
100
101
|
# 2550 Failover can cause the server to become a replica, need
|
101
102
|
# to disconnect and reopen the socket to get back to the primary.
|
102
103
|
# 4495 Use the same logic if we have a "Not enough replicas" error from the primary
|
103
|
-
|
104
|
+
# 4985 Use the same logic when a blocking command is force-unblocked
|
105
|
+
if retryable && ex.message =~ /READONLY|NOREPLICAS|UNBLOCKED/
|
104
106
|
conn.disconnect!
|
105
107
|
retryable = false
|
106
108
|
retry
|
data/sidekiq.gemspec
CHANGED
@@ -18,7 +18,7 @@ Gem::Specification.new do |gem|
|
|
18
18
|
"homepage_uri" => "https://sidekiq.org",
|
19
19
|
"bug_tracker_uri" => "https://github.com/mperham/sidekiq/issues",
|
20
20
|
"documentation_uri" => "https://github.com/mperham/sidekiq/wiki",
|
21
|
-
"changelog_uri" => "https://github.com/mperham/sidekiq/blob/
|
21
|
+
"changelog_uri" => "https://github.com/mperham/sidekiq/blob/main/Changes.md",
|
22
22
|
"source_code_uri" => "https://github.com/mperham/sidekiq"
|
23
23
|
}
|
24
24
|
|