sidekiq 6.0.7 → 6.4.0
Sign up to get free protection for your applications and to get access to all the features.
Potentially problematic release.
This version of sidekiq might be problematic. Click here for more details.
- checksums.yaml +4 -4
- data/Changes.md +160 -2
- data/LICENSE +3 -3
- data/README.md +5 -9
- data/bin/sidekiq +7 -2
- data/lib/generators/sidekiq/job_generator.rb +57 -0
- data/lib/generators/sidekiq/templates/{worker.rb.erb → job.rb.erb} +2 -2
- data/lib/generators/sidekiq/templates/{worker_spec.rb.erb → job_spec.rb.erb} +1 -1
- data/lib/generators/sidekiq/templates/{worker_test.rb.erb → job_test.rb.erb} +1 -1
- data/lib/sidekiq/api.rb +104 -60
- data/lib/sidekiq/cli.rb +36 -10
- data/lib/sidekiq/client.rb +12 -45
- data/lib/sidekiq/delay.rb +2 -0
- data/lib/sidekiq/extensions/action_mailer.rb +5 -4
- data/lib/sidekiq/extensions/active_record.rb +6 -5
- data/lib/sidekiq/extensions/class_methods.rb +7 -6
- data/lib/sidekiq/extensions/generic_proxy.rb +5 -3
- data/lib/sidekiq/fetch.rb +30 -21
- data/lib/sidekiq/job.rb +13 -0
- data/lib/sidekiq/job_logger.rb +1 -1
- data/lib/sidekiq/job_retry.rb +10 -11
- data/lib/sidekiq/job_util.rb +65 -0
- data/lib/sidekiq/launcher.rb +79 -21
- data/lib/sidekiq/logger.rb +3 -2
- data/lib/sidekiq/manager.rb +10 -12
- data/lib/sidekiq/middleware/chain.rb +6 -4
- data/lib/sidekiq/middleware/current_attributes.rb +57 -0
- data/lib/sidekiq/processor.rb +4 -4
- data/lib/sidekiq/rails.rb +27 -18
- data/lib/sidekiq/redis_connection.rb +14 -13
- data/lib/sidekiq/scheduled.rb +51 -16
- data/lib/sidekiq/sd_notify.rb +1 -1
- data/lib/sidekiq/testing.rb +2 -4
- data/lib/sidekiq/util.rb +41 -0
- data/lib/sidekiq/version.rb +1 -1
- data/lib/sidekiq/web/action.rb +2 -2
- data/lib/sidekiq/web/application.rb +21 -12
- data/lib/sidekiq/web/csrf_protection.rb +180 -0
- data/lib/sidekiq/web/helpers.rb +35 -29
- data/lib/sidekiq/web/router.rb +5 -2
- data/lib/sidekiq/web.rb +36 -72
- data/lib/sidekiq/worker.rb +129 -12
- data/lib/sidekiq.rb +12 -3
- data/sidekiq.gemspec +11 -4
- data/web/assets/images/apple-touch-icon.png +0 -0
- data/web/assets/javascripts/application.js +82 -66
- data/web/assets/javascripts/dashboard.js +51 -51
- data/web/assets/stylesheets/application-dark.css +64 -43
- data/web/assets/stylesheets/application-rtl.css +0 -4
- data/web/assets/stylesheets/application.css +41 -239
- data/web/locales/ar.yml +8 -2
- data/web/locales/en.yml +4 -1
- data/web/locales/es.yml +18 -2
- data/web/locales/fr.yml +8 -1
- data/web/locales/ja.yml +3 -0
- data/web/locales/lt.yml +1 -1
- data/web/locales/pl.yml +4 -4
- data/web/locales/ru.yml +4 -0
- data/web/views/_footer.erb +1 -1
- data/web/views/_job_info.erb +1 -1
- data/web/views/_poll_link.erb +2 -5
- data/web/views/_summary.erb +7 -7
- data/web/views/busy.erb +50 -19
- data/web/views/dashboard.erb +22 -14
- data/web/views/dead.erb +1 -1
- data/web/views/layout.erb +2 -1
- data/web/views/morgue.erb +6 -6
- data/web/views/queue.erb +11 -11
- data/web/views/queues.erb +4 -4
- data/web/views/retries.erb +7 -7
- data/web/views/retry.erb +1 -1
- data/web/views/scheduled.erb +1 -1
- metadata +24 -49
- data/.circleci/config.yml +0 -60
- data/.github/contributing.md +0 -32
- data/.github/issue_template.md +0 -11
- data/.gitignore +0 -13
- data/.standard.yml +0 -20
- data/3.0-Upgrade.md +0 -70
- data/4.0-Upgrade.md +0 -53
- data/5.0-Upgrade.md +0 -56
- data/6.0-Upgrade.md +0 -72
- data/COMM-LICENSE +0 -97
- data/Ent-2.0-Upgrade.md +0 -37
- data/Ent-Changes.md +0 -256
- data/Gemfile +0 -24
- data/Gemfile.lock +0 -208
- data/Pro-2.0-Upgrade.md +0 -138
- data/Pro-3.0-Upgrade.md +0 -44
- data/Pro-4.0-Upgrade.md +0 -35
- data/Pro-5.0-Upgrade.md +0 -25
- data/Pro-Changes.md +0 -782
- data/Rakefile +0 -10
- data/code_of_conduct.md +0 -50
- data/lib/generators/sidekiq/worker_generator.rb +0 -57
data/lib/sidekiq/worker.rb
CHANGED
@@ -9,6 +9,7 @@ module Sidekiq
|
|
9
9
|
#
|
10
10
|
# class HardWorker
|
11
11
|
# include Sidekiq::Worker
|
12
|
+
# sidekiq_options queue: 'critical', retry: 5
|
12
13
|
#
|
13
14
|
# def perform(*args)
|
14
15
|
# # do some work
|
@@ -20,6 +21,26 @@ module Sidekiq
|
|
20
21
|
# HardWorker.perform_async(1, 2, 3)
|
21
22
|
#
|
22
23
|
# Note that perform_async is a class method, perform is an instance method.
|
24
|
+
#
|
25
|
+
# Sidekiq::Worker also includes several APIs to provide compatibility with
|
26
|
+
# ActiveJob.
|
27
|
+
#
|
28
|
+
# class SomeWorker
|
29
|
+
# include Sidekiq::Worker
|
30
|
+
# queue_as :critical
|
31
|
+
#
|
32
|
+
# def perform(...)
|
33
|
+
# end
|
34
|
+
# end
|
35
|
+
#
|
36
|
+
# SomeWorker.set(wait_until: 1.hour).perform_async(123)
|
37
|
+
#
|
38
|
+
# Note that arguments passed to the job must still obey Sidekiq's
|
39
|
+
# best practice for simple, JSON-native data types. Sidekiq will not
|
40
|
+
# implement ActiveJob's more complex argument serialization. For
|
41
|
+
# this reason, we don't implement `perform_later` as our call semantics
|
42
|
+
# are very different.
|
43
|
+
#
|
23
44
|
module Worker
|
24
45
|
##
|
25
46
|
# The Options module is extracted so we can include it in ActiveJob::Base
|
@@ -150,33 +171,95 @@ module Sidekiq
|
|
150
171
|
# SomeWorker.set(queue: 'foo').perform_async(....)
|
151
172
|
#
|
152
173
|
class Setter
|
174
|
+
include Sidekiq::JobUtil
|
175
|
+
|
153
176
|
def initialize(klass, opts)
|
154
177
|
@klass = klass
|
155
178
|
@opts = opts
|
179
|
+
|
180
|
+
# ActiveJob compatibility
|
181
|
+
interval = @opts.delete(:wait_until) || @opts.delete(:wait)
|
182
|
+
at(interval) if interval
|
156
183
|
end
|
157
184
|
|
158
185
|
def set(options)
|
186
|
+
interval = options.delete(:wait_until) || options.delete(:wait)
|
159
187
|
@opts.merge!(options)
|
188
|
+
at(interval) if interval
|
160
189
|
self
|
161
190
|
end
|
162
191
|
|
163
192
|
def perform_async(*args)
|
164
|
-
@
|
193
|
+
if @opts["sync"] == true
|
194
|
+
perform_inline(*args)
|
195
|
+
else
|
196
|
+
@klass.client_push(@opts.merge("args" => args, "class" => @klass))
|
197
|
+
end
|
198
|
+
end
|
199
|
+
|
200
|
+
# Explicit inline execution of a job. Returns nil if the job did not
|
201
|
+
# execute, true otherwise.
|
202
|
+
def perform_inline(*args)
|
203
|
+
raw = @opts.merge("args" => args, "class" => @klass).transform_keys(&:to_s)
|
204
|
+
|
205
|
+
# validate and normalize payload
|
206
|
+
item = normalize_item(raw)
|
207
|
+
queue = item["queue"]
|
208
|
+
|
209
|
+
# run client-side middleware
|
210
|
+
result = Sidekiq.client_middleware.invoke(item["class"], item, queue, Sidekiq.redis_pool) do
|
211
|
+
item
|
212
|
+
end
|
213
|
+
return nil unless result
|
214
|
+
|
215
|
+
# round-trip the payload via JSON
|
216
|
+
msg = Sidekiq.load_json(Sidekiq.dump_json(item))
|
217
|
+
|
218
|
+
# prepare the job instance
|
219
|
+
klass = msg["class"].constantize
|
220
|
+
job = klass.new
|
221
|
+
job.jid = msg["jid"]
|
222
|
+
job.bid = msg["bid"] if job.respond_to?(:bid)
|
223
|
+
|
224
|
+
# run the job through server-side middleware
|
225
|
+
result = Sidekiq.server_middleware.invoke(job, msg, msg["queue"]) do
|
226
|
+
# perform it
|
227
|
+
job.perform(*msg["args"])
|
228
|
+
true
|
229
|
+
end
|
230
|
+
return nil unless result
|
231
|
+
# jobs do not return a result. they should store any
|
232
|
+
# modified state.
|
233
|
+
true
|
234
|
+
end
|
235
|
+
alias_method :perform_sync, :perform_inline
|
236
|
+
|
237
|
+
def perform_bulk(args, batch_size: 1_000)
|
238
|
+
hash = @opts.transform_keys(&:to_s)
|
239
|
+
result = args.each_slice(batch_size).flat_map do |slice|
|
240
|
+
Sidekiq::Client.push_bulk(hash.merge("class" => @klass, "args" => slice))
|
241
|
+
end
|
242
|
+
|
243
|
+
result.is_a?(Enumerator::Lazy) ? result.force : result
|
165
244
|
end
|
166
245
|
|
167
246
|
# +interval+ must be a timestamp, numeric or something that acts
|
168
247
|
# numeric (like an activesupport time interval).
|
169
248
|
def perform_in(interval, *args)
|
249
|
+
at(interval).perform_async(*args)
|
250
|
+
end
|
251
|
+
alias_method :perform_at, :perform_in
|
252
|
+
|
253
|
+
private
|
254
|
+
|
255
|
+
def at(interval)
|
170
256
|
int = interval.to_f
|
171
257
|
now = Time.now.to_f
|
172
258
|
ts = (int < 1_000_000_000 ? now + int : int)
|
173
|
-
|
174
|
-
payload = @opts.merge("class" => @klass, "args" => args)
|
175
259
|
# Optimization to enqueue something now that is scheduled to go out now or in the past
|
176
|
-
|
177
|
-
|
260
|
+
@opts["at"] = ts if ts > now
|
261
|
+
self
|
178
262
|
end
|
179
|
-
alias_method :perform_at, :perform_in
|
180
263
|
end
|
181
264
|
|
182
265
|
module ClassMethods
|
@@ -192,12 +275,49 @@ module Sidekiq
|
|
192
275
|
raise ArgumentError, "Do not call .delay_until on a Sidekiq::Worker class, call .perform_at"
|
193
276
|
end
|
194
277
|
|
278
|
+
def queue_as(q)
|
279
|
+
sidekiq_options("queue" => q.to_s)
|
280
|
+
end
|
281
|
+
|
195
282
|
def set(options)
|
196
283
|
Setter.new(self, options)
|
197
284
|
end
|
198
285
|
|
199
286
|
def perform_async(*args)
|
200
|
-
|
287
|
+
Setter.new(self, {}).perform_async(*args)
|
288
|
+
end
|
289
|
+
|
290
|
+
# Inline execution of job's perform method after passing through Sidekiq.client_middleware and Sidekiq.server_middleware
|
291
|
+
def perform_inline(*args)
|
292
|
+
Setter.new(self, {}).perform_inline(*args)
|
293
|
+
end
|
294
|
+
|
295
|
+
##
|
296
|
+
# Push a large number of jobs to Redis, while limiting the batch of
|
297
|
+
# each job payload to 1,000. This method helps cut down on the number
|
298
|
+
# of round trips to Redis, which can increase the performance of enqueueing
|
299
|
+
# large numbers of jobs.
|
300
|
+
#
|
301
|
+
# +items+ must be an Array of Arrays.
|
302
|
+
#
|
303
|
+
# For finer-grained control, use `Sidekiq::Client.push_bulk` directly.
|
304
|
+
#
|
305
|
+
# Example (3 Redis round trips):
|
306
|
+
#
|
307
|
+
# SomeWorker.perform_async(1)
|
308
|
+
# SomeWorker.perform_async(2)
|
309
|
+
# SomeWorker.perform_async(3)
|
310
|
+
#
|
311
|
+
# Would instead become (1 Redis round trip):
|
312
|
+
#
|
313
|
+
# SomeWorker.perform_bulk([[1], [2], [3]])
|
314
|
+
#
|
315
|
+
def perform_bulk(items, batch_size: 1_000)
|
316
|
+
result = items.each_slice(batch_size).flat_map do |slice|
|
317
|
+
Sidekiq::Client.push_bulk("class" => self, "args" => slice)
|
318
|
+
end
|
319
|
+
|
320
|
+
result.is_a?(Enumerator::Lazy) ? result.force : result
|
201
321
|
end
|
202
322
|
|
203
323
|
# +interval+ must be a timestamp, numeric or something that acts
|
@@ -235,12 +355,9 @@ module Sidekiq
|
|
235
355
|
|
236
356
|
def client_push(item) # :nodoc:
|
237
357
|
pool = Thread.current[:sidekiq_via_pool] || get_sidekiq_options["pool"] || Sidekiq.redis_pool
|
238
|
-
|
239
|
-
item.keys.each do |key|
|
240
|
-
item[key.to_s] = item.delete(key)
|
241
|
-
end
|
358
|
+
stringified_item = item.transform_keys(&:to_s)
|
242
359
|
|
243
|
-
Sidekiq::Client.new(pool).push(
|
360
|
+
Sidekiq::Client.new(pool).push(stringified_item)
|
244
361
|
end
|
245
362
|
end
|
246
363
|
end
|
data/lib/sidekiq.rb
CHANGED
@@ -6,6 +6,7 @@ fail "Sidekiq #{Sidekiq::VERSION} does not support Ruby versions below 2.5.0." i
|
|
6
6
|
require "sidekiq/logger"
|
7
7
|
require "sidekiq/client"
|
8
8
|
require "sidekiq/worker"
|
9
|
+
require "sidekiq/job"
|
9
10
|
require "sidekiq/redis_connection"
|
10
11
|
require "sidekiq/delay"
|
11
12
|
|
@@ -20,10 +21,12 @@ module Sidekiq
|
|
20
21
|
labels: [],
|
21
22
|
concurrency: 10,
|
22
23
|
require: ".",
|
24
|
+
strict: true,
|
23
25
|
environment: nil,
|
24
26
|
timeout: 25,
|
25
27
|
poll_interval_average: nil,
|
26
28
|
average_scheduled_poll_interval: 5,
|
29
|
+
on_complex_arguments: :warn,
|
27
30
|
error_handlers: [],
|
28
31
|
death_handlers: [],
|
29
32
|
lifecycle_events: {
|
@@ -95,10 +98,12 @@ module Sidekiq
|
|
95
98
|
retryable = true
|
96
99
|
begin
|
97
100
|
yield conn
|
98
|
-
rescue Redis::
|
101
|
+
rescue Redis::BaseError => ex
|
99
102
|
# 2550 Failover can cause the server to become a replica, need
|
100
103
|
# to disconnect and reopen the socket to get back to the primary.
|
101
|
-
if
|
104
|
+
# 4495 Use the same logic if we have a "Not enough replicas" error from the primary
|
105
|
+
# 4985 Use the same logic when a blocking command is force-unblocked
|
106
|
+
if retryable && ex.message =~ /READONLY|NOREPLICAS|UNBLOCKED/
|
102
107
|
conn.disconnect!
|
103
108
|
retryable = false
|
104
109
|
retry
|
@@ -196,7 +201,7 @@ module Sidekiq
|
|
196
201
|
end
|
197
202
|
|
198
203
|
def self.logger
|
199
|
-
@logger ||= Sidekiq::Logger.new(
|
204
|
+
@logger ||= Sidekiq::Logger.new($stdout, level: Logger::INFO)
|
200
205
|
end
|
201
206
|
|
202
207
|
def self.logger=(logger)
|
@@ -248,6 +253,10 @@ module Sidekiq
|
|
248
253
|
options[:lifecycle_events][event] << block
|
249
254
|
end
|
250
255
|
|
256
|
+
def self.strict_args!(mode = :raise)
|
257
|
+
options[:on_complex_arguments] = mode
|
258
|
+
end
|
259
|
+
|
251
260
|
# We are shutting down Sidekiq but what about workers that
|
252
261
|
# are working on some long job? This error is
|
253
262
|
# raised in workers that have not finished within the hard
|
data/sidekiq.gemspec
CHANGED
@@ -5,17 +5,24 @@ Gem::Specification.new do |gem|
|
|
5
5
|
gem.email = ["mperham@gmail.com"]
|
6
6
|
gem.summary = "Simple, efficient background processing for Ruby"
|
7
7
|
gem.description = "Simple, efficient background processing for Ruby."
|
8
|
-
gem.homepage = "
|
8
|
+
gem.homepage = "https://sidekiq.org"
|
9
9
|
gem.license = "LGPL-3.0"
|
10
10
|
|
11
11
|
gem.executables = ["sidekiq", "sidekiqmon"]
|
12
|
-
gem.files = `git ls-files | grep -
|
12
|
+
gem.files = ["sidekiq.gemspec", "README.md", "Changes.md", "LICENSE"] + `git ls-files | grep -E '^(bin|lib|web)'`.split("\n")
|
13
13
|
gem.name = "sidekiq"
|
14
14
|
gem.version = Sidekiq::VERSION
|
15
15
|
gem.required_ruby_version = ">= 2.5.0"
|
16
16
|
|
17
|
-
gem.
|
17
|
+
gem.metadata = {
|
18
|
+
"homepage_uri" => "https://sidekiq.org",
|
19
|
+
"bug_tracker_uri" => "https://github.com/mperham/sidekiq/issues",
|
20
|
+
"documentation_uri" => "https://github.com/mperham/sidekiq/wiki",
|
21
|
+
"changelog_uri" => "https://github.com/mperham/sidekiq/blob/main/Changes.md",
|
22
|
+
"source_code_uri" => "https://github.com/mperham/sidekiq"
|
23
|
+
}
|
24
|
+
|
25
|
+
gem.add_dependency "redis", ">= 4.2.0"
|
18
26
|
gem.add_dependency "connection_pool", ">= 2.2.2"
|
19
27
|
gem.add_dependency "rack", "~> 2.0"
|
20
|
-
gem.add_dependency "rack-protection", ">= 2.0.0"
|
21
28
|
end
|
Binary file
|