sidekiq 6.2.1 → 6.4.0
Sign up to get free protection for your applications and to get access to all the features.
Potentially problematic release.
This version of sidekiq might be problematic. Click here for more details.
- checksums.yaml +4 -4
- data/Changes.md +79 -1
- data/LICENSE +3 -3
- data/README.md +3 -3
- data/lib/generators/sidekiq/job_generator.rb +57 -0
- data/lib/generators/sidekiq/templates/{worker.rb.erb → job.rb.erb} +2 -2
- data/lib/generators/sidekiq/templates/{worker_spec.rb.erb → job_spec.rb.erb} +1 -1
- data/lib/generators/sidekiq/templates/{worker_test.rb.erb → job_test.rb.erb} +1 -1
- data/lib/sidekiq/api.rb +86 -60
- data/lib/sidekiq/cli.rb +13 -3
- data/lib/sidekiq/client.rb +5 -39
- data/lib/sidekiq/delay.rb +2 -0
- data/lib/sidekiq/extensions/action_mailer.rb +2 -2
- data/lib/sidekiq/extensions/active_record.rb +2 -2
- data/lib/sidekiq/extensions/class_methods.rb +2 -2
- data/lib/sidekiq/extensions/generic_proxy.rb +5 -3
- data/lib/sidekiq/fetch.rb +5 -4
- data/lib/sidekiq/job.rb +13 -0
- data/lib/sidekiq/job_logger.rb +1 -1
- data/lib/sidekiq/job_retry.rb +9 -11
- data/lib/sidekiq/job_util.rb +65 -0
- data/lib/sidekiq/launcher.rb +18 -18
- data/lib/sidekiq/manager.rb +7 -9
- data/lib/sidekiq/middleware/chain.rb +5 -3
- data/lib/sidekiq/middleware/current_attributes.rb +57 -0
- data/lib/sidekiq/rails.rb +11 -0
- data/lib/sidekiq/redis_connection.rb +4 -6
- data/lib/sidekiq/scheduled.rb +51 -16
- data/lib/sidekiq/testing.rb +1 -3
- data/lib/sidekiq/util.rb +13 -0
- data/lib/sidekiq/version.rb +1 -1
- data/lib/sidekiq/web/action.rb +1 -1
- data/lib/sidekiq/web/application.rb +9 -6
- data/lib/sidekiq/web/helpers.rb +9 -21
- data/lib/sidekiq/web.rb +4 -3
- data/lib/sidekiq/worker.rb +127 -7
- data/lib/sidekiq.rb +8 -1
- data/sidekiq.gemspec +1 -1
- data/web/assets/javascripts/application.js +82 -61
- data/web/assets/javascripts/dashboard.js +51 -51
- data/web/assets/stylesheets/application-dark.css +28 -45
- data/web/assets/stylesheets/application-rtl.css +0 -4
- data/web/assets/stylesheets/application.css +23 -237
- data/web/locales/ar.yml +8 -2
- data/web/locales/en.yml +4 -1
- data/web/locales/es.yml +18 -2
- data/web/locales/fr.yml +7 -0
- data/web/locales/ja.yml +3 -0
- data/web/locales/lt.yml +1 -1
- data/web/views/_footer.erb +1 -1
- data/web/views/_job_info.erb +1 -1
- data/web/views/_poll_link.erb +2 -5
- data/web/views/_summary.erb +7 -7
- data/web/views/busy.erb +5 -5
- data/web/views/dashboard.erb +22 -14
- data/web/views/dead.erb +1 -1
- data/web/views/layout.erb +1 -1
- data/web/views/morgue.erb +6 -6
- data/web/views/queue.erb +10 -10
- data/web/views/queues.erb +3 -3
- data/web/views/retries.erb +7 -7
- data/web/views/retry.erb +1 -1
- data/web/views/scheduled.erb +1 -1
- metadata +10 -7
- data/lib/generators/sidekiq/worker_generator.rb +0 -57
data/lib/sidekiq/web/helpers.rb
CHANGED
@@ -10,14 +10,13 @@ module Sidekiq
|
|
10
10
|
module WebHelpers
|
11
11
|
def strings(lang)
|
12
12
|
@strings ||= {}
|
13
|
-
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
end
|
13
|
+
|
14
|
+
# Allow sidekiq-web extensions to add locale paths
|
15
|
+
# so extensions can be localized
|
16
|
+
@strings[lang] ||= settings.locales.each_with_object({}) do |path, global|
|
17
|
+
find_locale_files(lang).each do |file|
|
18
|
+
strs = YAML.load(File.open(file))
|
19
|
+
global.merge!(strs[lang])
|
21
20
|
end
|
22
21
|
end
|
23
22
|
end
|
@@ -71,17 +70,6 @@ module Sidekiq
|
|
71
70
|
@head_html.join if defined?(@head_html)
|
72
71
|
end
|
73
72
|
|
74
|
-
def poll_path
|
75
|
-
if current_path != "" && params["poll"]
|
76
|
-
path = root_path + current_path
|
77
|
-
query_string = to_query_string(params.slice(*params.keys - %w[page poll]))
|
78
|
-
path += "?#{query_string}" unless query_string.empty?
|
79
|
-
path
|
80
|
-
else
|
81
|
-
""
|
82
|
-
end
|
83
|
-
end
|
84
|
-
|
85
73
|
def text_direction
|
86
74
|
get_locale["TextDirection"] || "ltr"
|
87
75
|
end
|
@@ -126,7 +114,7 @@ module Sidekiq
|
|
126
114
|
# within is used by Sidekiq Pro
|
127
115
|
def display_tags(job, within = nil)
|
128
116
|
job.tags.map { |tag|
|
129
|
-
"<span class='
|
117
|
+
"<span class='label label-info jobtag'>#{::Rack::Utils.escape_html(tag)}</span>"
|
130
118
|
}.join(" ")
|
131
119
|
end
|
132
120
|
|
@@ -204,7 +192,7 @@ module Sidekiq
|
|
204
192
|
[score.to_f, jid]
|
205
193
|
end
|
206
194
|
|
207
|
-
SAFE_QPARAMS = %w[page
|
195
|
+
SAFE_QPARAMS = %w[page direction]
|
208
196
|
|
209
197
|
# Merge options with current params, filter safe params, and stringify to query string
|
210
198
|
def qparams(options)
|
data/lib/sidekiq/web.rb
CHANGED
@@ -143,13 +143,14 @@ module Sidekiq
|
|
143
143
|
klass = self.class
|
144
144
|
m = middlewares
|
145
145
|
|
146
|
+
rules = []
|
147
|
+
rules = [[:all, {"Cache-Control" => "public, max-age=86400"}]] unless ENV["SIDEKIQ_WEB_TESTING"]
|
148
|
+
|
146
149
|
::Rack::Builder.new do
|
147
150
|
use Rack::Static, urls: ["/stylesheets", "/images", "/javascripts"],
|
148
151
|
root: ASSETS,
|
149
152
|
cascade: true,
|
150
|
-
header_rules:
|
151
|
-
[:all, {"Cache-Control" => "public, max-age=86400"}]
|
152
|
-
]
|
153
|
+
header_rules: rules
|
153
154
|
m.each { |middleware, block| use(*middleware, &block) }
|
154
155
|
use Sidekiq::Web::CsrfProtection unless $TESTING
|
155
156
|
run WebApplication.new(klass)
|
data/lib/sidekiq/worker.rb
CHANGED
@@ -9,6 +9,7 @@ module Sidekiq
|
|
9
9
|
#
|
10
10
|
# class HardWorker
|
11
11
|
# include Sidekiq::Worker
|
12
|
+
# sidekiq_options queue: 'critical', retry: 5
|
12
13
|
#
|
13
14
|
# def perform(*args)
|
14
15
|
# # do some work
|
@@ -20,6 +21,26 @@ module Sidekiq
|
|
20
21
|
# HardWorker.perform_async(1, 2, 3)
|
21
22
|
#
|
22
23
|
# Note that perform_async is a class method, perform is an instance method.
|
24
|
+
#
|
25
|
+
# Sidekiq::Worker also includes several APIs to provide compatibility with
|
26
|
+
# ActiveJob.
|
27
|
+
#
|
28
|
+
# class SomeWorker
|
29
|
+
# include Sidekiq::Worker
|
30
|
+
# queue_as :critical
|
31
|
+
#
|
32
|
+
# def perform(...)
|
33
|
+
# end
|
34
|
+
# end
|
35
|
+
#
|
36
|
+
# SomeWorker.set(wait_until: 1.hour).perform_async(123)
|
37
|
+
#
|
38
|
+
# Note that arguments passed to the job must still obey Sidekiq's
|
39
|
+
# best practice for simple, JSON-native data types. Sidekiq will not
|
40
|
+
# implement ActiveJob's more complex argument serialization. For
|
41
|
+
# this reason, we don't implement `perform_later` as our call semantics
|
42
|
+
# are very different.
|
43
|
+
#
|
23
44
|
module Worker
|
24
45
|
##
|
25
46
|
# The Options module is extracted so we can include it in ActiveJob::Base
|
@@ -150,33 +171,95 @@ module Sidekiq
|
|
150
171
|
# SomeWorker.set(queue: 'foo').perform_async(....)
|
151
172
|
#
|
152
173
|
class Setter
|
174
|
+
include Sidekiq::JobUtil
|
175
|
+
|
153
176
|
def initialize(klass, opts)
|
154
177
|
@klass = klass
|
155
178
|
@opts = opts
|
179
|
+
|
180
|
+
# ActiveJob compatibility
|
181
|
+
interval = @opts.delete(:wait_until) || @opts.delete(:wait)
|
182
|
+
at(interval) if interval
|
156
183
|
end
|
157
184
|
|
158
185
|
def set(options)
|
186
|
+
interval = options.delete(:wait_until) || options.delete(:wait)
|
159
187
|
@opts.merge!(options)
|
188
|
+
at(interval) if interval
|
160
189
|
self
|
161
190
|
end
|
162
191
|
|
163
192
|
def perform_async(*args)
|
164
|
-
@
|
193
|
+
if @opts["sync"] == true
|
194
|
+
perform_inline(*args)
|
195
|
+
else
|
196
|
+
@klass.client_push(@opts.merge("args" => args, "class" => @klass))
|
197
|
+
end
|
198
|
+
end
|
199
|
+
|
200
|
+
# Explicit inline execution of a job. Returns nil if the job did not
|
201
|
+
# execute, true otherwise.
|
202
|
+
def perform_inline(*args)
|
203
|
+
raw = @opts.merge("args" => args, "class" => @klass).transform_keys(&:to_s)
|
204
|
+
|
205
|
+
# validate and normalize payload
|
206
|
+
item = normalize_item(raw)
|
207
|
+
queue = item["queue"]
|
208
|
+
|
209
|
+
# run client-side middleware
|
210
|
+
result = Sidekiq.client_middleware.invoke(item["class"], item, queue, Sidekiq.redis_pool) do
|
211
|
+
item
|
212
|
+
end
|
213
|
+
return nil unless result
|
214
|
+
|
215
|
+
# round-trip the payload via JSON
|
216
|
+
msg = Sidekiq.load_json(Sidekiq.dump_json(item))
|
217
|
+
|
218
|
+
# prepare the job instance
|
219
|
+
klass = msg["class"].constantize
|
220
|
+
job = klass.new
|
221
|
+
job.jid = msg["jid"]
|
222
|
+
job.bid = msg["bid"] if job.respond_to?(:bid)
|
223
|
+
|
224
|
+
# run the job through server-side middleware
|
225
|
+
result = Sidekiq.server_middleware.invoke(job, msg, msg["queue"]) do
|
226
|
+
# perform it
|
227
|
+
job.perform(*msg["args"])
|
228
|
+
true
|
229
|
+
end
|
230
|
+
return nil unless result
|
231
|
+
# jobs do not return a result. they should store any
|
232
|
+
# modified state.
|
233
|
+
true
|
234
|
+
end
|
235
|
+
alias_method :perform_sync, :perform_inline
|
236
|
+
|
237
|
+
def perform_bulk(args, batch_size: 1_000)
|
238
|
+
hash = @opts.transform_keys(&:to_s)
|
239
|
+
result = args.each_slice(batch_size).flat_map do |slice|
|
240
|
+
Sidekiq::Client.push_bulk(hash.merge("class" => @klass, "args" => slice))
|
241
|
+
end
|
242
|
+
|
243
|
+
result.is_a?(Enumerator::Lazy) ? result.force : result
|
165
244
|
end
|
166
245
|
|
167
246
|
# +interval+ must be a timestamp, numeric or something that acts
|
168
247
|
# numeric (like an activesupport time interval).
|
169
248
|
def perform_in(interval, *args)
|
249
|
+
at(interval).perform_async(*args)
|
250
|
+
end
|
251
|
+
alias_method :perform_at, :perform_in
|
252
|
+
|
253
|
+
private
|
254
|
+
|
255
|
+
def at(interval)
|
170
256
|
int = interval.to_f
|
171
257
|
now = Time.now.to_f
|
172
258
|
ts = (int < 1_000_000_000 ? now + int : int)
|
173
|
-
|
174
|
-
payload = @opts.merge("class" => @klass, "args" => args)
|
175
259
|
# Optimization to enqueue something now that is scheduled to go out now or in the past
|
176
|
-
|
177
|
-
|
260
|
+
@opts["at"] = ts if ts > now
|
261
|
+
self
|
178
262
|
end
|
179
|
-
alias_method :perform_at, :perform_in
|
180
263
|
end
|
181
264
|
|
182
265
|
module ClassMethods
|
@@ -192,12 +275,49 @@ module Sidekiq
|
|
192
275
|
raise ArgumentError, "Do not call .delay_until on a Sidekiq::Worker class, call .perform_at"
|
193
276
|
end
|
194
277
|
|
278
|
+
def queue_as(q)
|
279
|
+
sidekiq_options("queue" => q.to_s)
|
280
|
+
end
|
281
|
+
|
195
282
|
def set(options)
|
196
283
|
Setter.new(self, options)
|
197
284
|
end
|
198
285
|
|
199
286
|
def perform_async(*args)
|
200
|
-
|
287
|
+
Setter.new(self, {}).perform_async(*args)
|
288
|
+
end
|
289
|
+
|
290
|
+
# Inline execution of job's perform method after passing through Sidekiq.client_middleware and Sidekiq.server_middleware
|
291
|
+
def perform_inline(*args)
|
292
|
+
Setter.new(self, {}).perform_inline(*args)
|
293
|
+
end
|
294
|
+
|
295
|
+
##
|
296
|
+
# Push a large number of jobs to Redis, while limiting the batch of
|
297
|
+
# each job payload to 1,000. This method helps cut down on the number
|
298
|
+
# of round trips to Redis, which can increase the performance of enqueueing
|
299
|
+
# large numbers of jobs.
|
300
|
+
#
|
301
|
+
# +items+ must be an Array of Arrays.
|
302
|
+
#
|
303
|
+
# For finer-grained control, use `Sidekiq::Client.push_bulk` directly.
|
304
|
+
#
|
305
|
+
# Example (3 Redis round trips):
|
306
|
+
#
|
307
|
+
# SomeWorker.perform_async(1)
|
308
|
+
# SomeWorker.perform_async(2)
|
309
|
+
# SomeWorker.perform_async(3)
|
310
|
+
#
|
311
|
+
# Would instead become (1 Redis round trip):
|
312
|
+
#
|
313
|
+
# SomeWorker.perform_bulk([[1], [2], [3]])
|
314
|
+
#
|
315
|
+
def perform_bulk(items, batch_size: 1_000)
|
316
|
+
result = items.each_slice(batch_size).flat_map do |slice|
|
317
|
+
Sidekiq::Client.push_bulk("class" => self, "args" => slice)
|
318
|
+
end
|
319
|
+
|
320
|
+
result.is_a?(Enumerator::Lazy) ? result.force : result
|
201
321
|
end
|
202
322
|
|
203
323
|
# +interval+ must be a timestamp, numeric or something that acts
|
data/lib/sidekiq.rb
CHANGED
@@ -6,6 +6,7 @@ fail "Sidekiq #{Sidekiq::VERSION} does not support Ruby versions below 2.5.0." i
|
|
6
6
|
require "sidekiq/logger"
|
7
7
|
require "sidekiq/client"
|
8
8
|
require "sidekiq/worker"
|
9
|
+
require "sidekiq/job"
|
9
10
|
require "sidekiq/redis_connection"
|
10
11
|
require "sidekiq/delay"
|
11
12
|
|
@@ -25,6 +26,7 @@ module Sidekiq
|
|
25
26
|
timeout: 25,
|
26
27
|
poll_interval_average: nil,
|
27
28
|
average_scheduled_poll_interval: 5,
|
29
|
+
on_complex_arguments: :warn,
|
28
30
|
error_handlers: [],
|
29
31
|
death_handlers: [],
|
30
32
|
lifecycle_events: {
|
@@ -100,7 +102,8 @@ module Sidekiq
|
|
100
102
|
# 2550 Failover can cause the server to become a replica, need
|
101
103
|
# to disconnect and reopen the socket to get back to the primary.
|
102
104
|
# 4495 Use the same logic if we have a "Not enough replicas" error from the primary
|
103
|
-
|
105
|
+
# 4985 Use the same logic when a blocking command is force-unblocked
|
106
|
+
if retryable && ex.message =~ /READONLY|NOREPLICAS|UNBLOCKED/
|
104
107
|
conn.disconnect!
|
105
108
|
retryable = false
|
106
109
|
retry
|
@@ -250,6 +253,10 @@ module Sidekiq
|
|
250
253
|
options[:lifecycle_events][event] << block
|
251
254
|
end
|
252
255
|
|
256
|
+
def self.strict_args!(mode = :raise)
|
257
|
+
options[:on_complex_arguments] = mode
|
258
|
+
end
|
259
|
+
|
253
260
|
# We are shutting down Sidekiq but what about workers that
|
254
261
|
# are working on some long job? This error is
|
255
262
|
# raised in workers that have not finished within the hard
|
data/sidekiq.gemspec
CHANGED
@@ -18,7 +18,7 @@ Gem::Specification.new do |gem|
|
|
18
18
|
"homepage_uri" => "https://sidekiq.org",
|
19
19
|
"bug_tracker_uri" => "https://github.com/mperham/sidekiq/issues",
|
20
20
|
"documentation_uri" => "https://github.com/mperham/sidekiq/wiki",
|
21
|
-
"changelog_uri" => "https://github.com/mperham/sidekiq/blob/
|
21
|
+
"changelog_uri" => "https://github.com/mperham/sidekiq/blob/main/Changes.md",
|
22
22
|
"source_code_uri" => "https://github.com/mperham/sidekiq"
|
23
23
|
}
|
24
24
|
|