sidekiq 6.0.0 → 6.3.0
Sign up to get free protection for your applications and to get access to all the features.
Potentially problematic release.
This version of sidekiq might be problematic. Click here for more details.
- checksums.yaml +4 -4
- data/Changes.md +258 -2
- data/LICENSE +1 -1
- data/README.md +6 -8
- data/bin/sidekiq +26 -2
- data/bin/sidekiqload +8 -4
- data/bin/sidekiqmon +4 -5
- data/lib/generators/sidekiq/worker_generator.rb +11 -1
- data/lib/sidekiq/api.rb +220 -145
- data/lib/sidekiq/cli.rb +64 -27
- data/lib/sidekiq/client.rb +31 -14
- data/lib/sidekiq/extensions/action_mailer.rb +3 -2
- data/lib/sidekiq/extensions/active_record.rb +4 -3
- data/lib/sidekiq/extensions/class_methods.rb +5 -4
- data/lib/sidekiq/extensions/generic_proxy.rb +3 -1
- data/lib/sidekiq/fetch.rb +36 -27
- data/lib/sidekiq/job.rb +13 -0
- data/lib/sidekiq/job_logger.rb +13 -5
- data/lib/sidekiq/job_retry.rb +27 -17
- data/lib/sidekiq/launcher.rb +110 -28
- data/lib/sidekiq/logger.rb +109 -12
- data/lib/sidekiq/manager.rb +4 -4
- data/lib/sidekiq/middleware/chain.rb +17 -6
- data/lib/sidekiq/middleware/current_attributes.rb +48 -0
- data/lib/sidekiq/monitor.rb +3 -18
- data/lib/sidekiq/paginator.rb +7 -2
- data/lib/sidekiq/processor.rb +22 -24
- data/lib/sidekiq/rails.rb +27 -18
- data/lib/sidekiq/redis_connection.rb +19 -13
- data/lib/sidekiq/scheduled.rb +37 -11
- data/lib/sidekiq/sd_notify.rb +149 -0
- data/lib/sidekiq/systemd.rb +24 -0
- data/lib/sidekiq/testing.rb +14 -4
- data/lib/sidekiq/util.rb +28 -2
- data/lib/sidekiq/version.rb +1 -1
- data/lib/sidekiq/web/action.rb +2 -2
- data/lib/sidekiq/web/application.rb +37 -30
- data/lib/sidekiq/web/csrf_protection.rb +180 -0
- data/lib/sidekiq/web/helpers.rb +51 -33
- data/lib/sidekiq/web/router.rb +6 -5
- data/lib/sidekiq/web.rb +37 -73
- data/lib/sidekiq/worker.rb +78 -14
- data/lib/sidekiq.rb +24 -8
- data/sidekiq.gemspec +13 -6
- data/web/assets/images/apple-touch-icon.png +0 -0
- data/web/assets/javascripts/application.js +83 -64
- data/web/assets/javascripts/dashboard.js +53 -53
- data/web/assets/stylesheets/application-dark.css +147 -0
- data/web/assets/stylesheets/application-rtl.css +0 -4
- data/web/assets/stylesheets/application.css +43 -230
- data/web/locales/ar.yml +8 -2
- data/web/locales/de.yml +14 -2
- data/web/locales/en.yml +6 -1
- data/web/locales/es.yml +18 -2
- data/web/locales/fr.yml +10 -3
- data/web/locales/ja.yml +5 -0
- data/web/locales/lt.yml +83 -0
- data/web/locales/pl.yml +4 -4
- data/web/locales/ru.yml +4 -0
- data/web/locales/vi.yml +83 -0
- data/web/views/_footer.erb +1 -1
- data/web/views/_job_info.erb +3 -2
- data/web/views/_poll_link.erb +2 -5
- data/web/views/_summary.erb +7 -7
- data/web/views/busy.erb +54 -20
- data/web/views/dashboard.erb +22 -14
- data/web/views/dead.erb +3 -3
- data/web/views/layout.erb +3 -1
- data/web/views/morgue.erb +9 -6
- data/web/views/queue.erb +19 -10
- data/web/views/queues.erb +10 -2
- data/web/views/retries.erb +11 -8
- data/web/views/retry.erb +3 -3
- data/web/views/scheduled.erb +5 -2
- metadata +29 -50
- data/.circleci/config.yml +0 -61
- data/.github/contributing.md +0 -32
- data/.github/issue_template.md +0 -11
- data/.gitignore +0 -13
- data/.standard.yml +0 -20
- data/3.0-Upgrade.md +0 -70
- data/4.0-Upgrade.md +0 -53
- data/5.0-Upgrade.md +0 -56
- data/6.0-Upgrade.md +0 -70
- data/COMM-LICENSE +0 -97
- data/Ent-2.0-Upgrade.md +0 -37
- data/Ent-Changes.md +0 -250
- data/Gemfile +0 -24
- data/Gemfile.lock +0 -196
- data/Pro-2.0-Upgrade.md +0 -138
- data/Pro-3.0-Upgrade.md +0 -44
- data/Pro-4.0-Upgrade.md +0 -35
- data/Pro-5.0-Upgrade.md +0 -25
- data/Pro-Changes.md +0 -768
- data/Rakefile +0 -10
- data/code_of_conduct.md +0 -50
data/lib/sidekiq/worker.rb
CHANGED
@@ -9,6 +9,7 @@ module Sidekiq
|
|
9
9
|
#
|
10
10
|
# class HardWorker
|
11
11
|
# include Sidekiq::Worker
|
12
|
+
# sidekiq_options queue: 'critical', retry: 5
|
12
13
|
#
|
13
14
|
# def perform(*args)
|
14
15
|
# # do some work
|
@@ -20,6 +21,26 @@ module Sidekiq
|
|
20
21
|
# HardWorker.perform_async(1, 2, 3)
|
21
22
|
#
|
22
23
|
# Note that perform_async is a class method, perform is an instance method.
|
24
|
+
#
|
25
|
+
# Sidekiq::Worker also includes several APIs to provide compatibility with
|
26
|
+
# ActiveJob.
|
27
|
+
#
|
28
|
+
# class SomeWorker
|
29
|
+
# include Sidekiq::Worker
|
30
|
+
# queue_as :critical
|
31
|
+
#
|
32
|
+
# def perform(...)
|
33
|
+
# end
|
34
|
+
# end
|
35
|
+
#
|
36
|
+
# SomeWorker.set(wait_until: 1.hour).perform_async(123)
|
37
|
+
#
|
38
|
+
# Note that arguments passed to the job must still obey Sidekiq's
|
39
|
+
# best practice for simple, JSON-native data types. Sidekiq will not
|
40
|
+
# implement ActiveJob's more complex argument serialization. For
|
41
|
+
# this reason, we don't implement `perform_later` as our call semantics
|
42
|
+
# are very different.
|
43
|
+
#
|
23
44
|
module Worker
|
24
45
|
##
|
25
46
|
# The Options module is extracted so we can include it in ActiveJob::Base
|
@@ -48,8 +69,8 @@ module Sidekiq
|
|
48
69
|
# In practice, any option is allowed. This is the main mechanism to configure the
|
49
70
|
# options for a specific job.
|
50
71
|
def sidekiq_options(opts = {})
|
51
|
-
opts =
|
52
|
-
self.sidekiq_options_hash = get_sidekiq_options.merge(
|
72
|
+
opts = opts.transform_keys(&:to_s) # stringify
|
73
|
+
self.sidekiq_options_hash = get_sidekiq_options.merge(opts)
|
53
74
|
end
|
54
75
|
|
55
76
|
def sidekiq_retry_in(&block)
|
@@ -153,10 +174,16 @@ module Sidekiq
|
|
153
174
|
def initialize(klass, opts)
|
154
175
|
@klass = klass
|
155
176
|
@opts = opts
|
177
|
+
|
178
|
+
# ActiveJob compatibility
|
179
|
+
interval = @opts.delete(:wait_until) || @opts.delete(:wait)
|
180
|
+
at(interval) if interval
|
156
181
|
end
|
157
182
|
|
158
183
|
def set(options)
|
184
|
+
interval = options.delete(:wait_until) || options.delete(:wait)
|
159
185
|
@opts.merge!(options)
|
186
|
+
at(interval) if interval
|
160
187
|
self
|
161
188
|
end
|
162
189
|
|
@@ -164,19 +191,29 @@ module Sidekiq
|
|
164
191
|
@klass.client_push(@opts.merge("args" => args, "class" => @klass))
|
165
192
|
end
|
166
193
|
|
194
|
+
def perform_bulk(args, batch_size: 1_000)
|
195
|
+
args.each_slice(batch_size).flat_map do |slice|
|
196
|
+
Sidekiq::Client.push_bulk(@opts.merge("class" => @klass, "args" => slice))
|
197
|
+
end
|
198
|
+
end
|
199
|
+
|
167
200
|
# +interval+ must be a timestamp, numeric or something that acts
|
168
201
|
# numeric (like an activesupport time interval).
|
169
202
|
def perform_in(interval, *args)
|
203
|
+
at(interval).perform_async(*args)
|
204
|
+
end
|
205
|
+
alias_method :perform_at, :perform_in
|
206
|
+
|
207
|
+
private
|
208
|
+
|
209
|
+
def at(interval)
|
170
210
|
int = interval.to_f
|
171
211
|
now = Time.now.to_f
|
172
212
|
ts = (int < 1_000_000_000 ? now + int : int)
|
173
|
-
|
174
|
-
payload = @opts.merge("class" => @klass, "args" => args, "at" => ts)
|
175
213
|
# Optimization to enqueue something now that is scheduled to go out now or in the past
|
176
|
-
|
177
|
-
|
214
|
+
@opts["at"] = ts if ts > now
|
215
|
+
self
|
178
216
|
end
|
179
|
-
alias_method :perform_at, :perform_in
|
180
217
|
end
|
181
218
|
|
182
219
|
module ClassMethods
|
@@ -192,6 +229,10 @@ module Sidekiq
|
|
192
229
|
raise ArgumentError, "Do not call .delay_until on a Sidekiq::Worker class, call .perform_at"
|
193
230
|
end
|
194
231
|
|
232
|
+
def queue_as(q)
|
233
|
+
sidekiq_options("queue" => q.to_s)
|
234
|
+
end
|
235
|
+
|
195
236
|
def set(options)
|
196
237
|
Setter.new(self, options)
|
197
238
|
end
|
@@ -200,6 +241,32 @@ module Sidekiq
|
|
200
241
|
client_push("class" => self, "args" => args)
|
201
242
|
end
|
202
243
|
|
244
|
+
##
|
245
|
+
# Push a large number of jobs to Redis, while limiting the batch of
|
246
|
+
# each job payload to 1,000. This method helps cut down on the number
|
247
|
+
# of round trips to Redis, which can increase the performance of enqueueing
|
248
|
+
# large numbers of jobs.
|
249
|
+
#
|
250
|
+
# +items+ must be an Array of Arrays.
|
251
|
+
#
|
252
|
+
# For finer-grained control, use `Sidekiq::Client.push_bulk` directly.
|
253
|
+
#
|
254
|
+
# Example (3 Redis round trips):
|
255
|
+
#
|
256
|
+
# SomeWorker.perform_async(1)
|
257
|
+
# SomeWorker.perform_async(2)
|
258
|
+
# SomeWorker.perform_async(3)
|
259
|
+
#
|
260
|
+
# Would instead become (1 Redis round trip):
|
261
|
+
#
|
262
|
+
# SomeWorker.perform_bulk([[1], [2], [3]])
|
263
|
+
#
|
264
|
+
def perform_bulk(items, batch_size: 1_000)
|
265
|
+
items.each_slice(batch_size).flat_map do |slice|
|
266
|
+
Sidekiq::Client.push_bulk("class" => self, "args" => slice)
|
267
|
+
end
|
268
|
+
end
|
269
|
+
|
203
270
|
# +interval+ must be a timestamp, numeric or something that acts
|
204
271
|
# numeric (like an activesupport time interval).
|
205
272
|
def perform_in(interval, *args)
|
@@ -207,10 +274,10 @@ module Sidekiq
|
|
207
274
|
now = Time.now.to_f
|
208
275
|
ts = (int < 1_000_000_000 ? now + int : int)
|
209
276
|
|
210
|
-
item = {"class" => self, "args" => args
|
277
|
+
item = {"class" => self, "args" => args}
|
211
278
|
|
212
279
|
# Optimization to enqueue something now that is scheduled to go out now or in the past
|
213
|
-
item
|
280
|
+
item["at"] = ts if ts > now
|
214
281
|
|
215
282
|
client_push(item)
|
216
283
|
end
|
@@ -235,12 +302,9 @@ module Sidekiq
|
|
235
302
|
|
236
303
|
def client_push(item) # :nodoc:
|
237
304
|
pool = Thread.current[:sidekiq_via_pool] || get_sidekiq_options["pool"] || Sidekiq.redis_pool
|
238
|
-
|
239
|
-
item.keys.each do |key|
|
240
|
-
item[key.to_s] = item.delete(key)
|
241
|
-
end
|
305
|
+
stringified_item = item.transform_keys(&:to_s)
|
242
306
|
|
243
|
-
Sidekiq::Client.new(pool).push(
|
307
|
+
Sidekiq::Client.new(pool).push(stringified_item)
|
244
308
|
end
|
245
309
|
end
|
246
310
|
end
|
data/lib/sidekiq.rb
CHANGED
@@ -6,6 +6,7 @@ fail "Sidekiq #{Sidekiq::VERSION} does not support Ruby versions below 2.5.0." i
|
|
6
6
|
require "sidekiq/logger"
|
7
7
|
require "sidekiq/client"
|
8
8
|
require "sidekiq/worker"
|
9
|
+
require "sidekiq/job"
|
9
10
|
require "sidekiq/redis_connection"
|
10
11
|
require "sidekiq/delay"
|
11
12
|
|
@@ -20,6 +21,7 @@ module Sidekiq
|
|
20
21
|
labels: [],
|
21
22
|
concurrency: 10,
|
22
23
|
require: ".",
|
24
|
+
strict: true,
|
23
25
|
environment: nil,
|
24
26
|
timeout: 25,
|
25
27
|
poll_interval_average: nil,
|
@@ -30,16 +32,16 @@ module Sidekiq
|
|
30
32
|
startup: [],
|
31
33
|
quiet: [],
|
32
34
|
shutdown: [],
|
33
|
-
heartbeat: []
|
35
|
+
heartbeat: []
|
34
36
|
},
|
35
37
|
dead_max_jobs: 10_000,
|
36
38
|
dead_timeout_in_seconds: 180 * 24 * 60 * 60, # 6 months
|
37
|
-
reloader: proc { |&block| block.call }
|
39
|
+
reloader: proc { |&block| block.call }
|
38
40
|
}
|
39
41
|
|
40
42
|
DEFAULT_WORKER_OPTIONS = {
|
41
43
|
"retry" => true,
|
42
|
-
"queue" => "default"
|
44
|
+
"queue" => "default"
|
43
45
|
}
|
44
46
|
|
45
47
|
FAKE_INFO = {
|
@@ -47,7 +49,7 @@ module Sidekiq
|
|
47
49
|
"uptime_in_days" => "9999",
|
48
50
|
"connected_clients" => "9999",
|
49
51
|
"used_memory_human" => "9P",
|
50
|
-
"used_memory_peak_human" => "9P"
|
52
|
+
"used_memory_peak_human" => "9P"
|
51
53
|
}
|
52
54
|
|
53
55
|
def self.❨╯°□°❩╯︵┻━┻
|
@@ -95,10 +97,12 @@ module Sidekiq
|
|
95
97
|
retryable = true
|
96
98
|
begin
|
97
99
|
yield conn
|
98
|
-
rescue Redis::
|
100
|
+
rescue Redis::BaseError => ex
|
99
101
|
# 2550 Failover can cause the server to become a replica, need
|
100
102
|
# to disconnect and reopen the socket to get back to the primary.
|
101
|
-
if
|
103
|
+
# 4495 Use the same logic if we have a "Not enough replicas" error from the primary
|
104
|
+
# 4985 Use the same logic when a blocking command is force-unblocked
|
105
|
+
if retryable && ex.message =~ /READONLY|NOREPLICAS|UNBLOCKED/
|
102
106
|
conn.disconnect!
|
103
107
|
retryable = false
|
104
108
|
retry
|
@@ -154,7 +158,7 @@ module Sidekiq
|
|
154
158
|
|
155
159
|
def self.default_worker_options=(hash)
|
156
160
|
# stringify
|
157
|
-
@default_worker_options = default_worker_options.merge(
|
161
|
+
@default_worker_options = default_worker_options.merge(hash.transform_keys(&:to_s))
|
158
162
|
end
|
159
163
|
|
160
164
|
def self.default_worker_options
|
@@ -192,16 +196,28 @@ module Sidekiq
|
|
192
196
|
|
193
197
|
def self.log_formatter=(log_formatter)
|
194
198
|
@log_formatter = log_formatter
|
199
|
+
logger.formatter = log_formatter
|
195
200
|
end
|
196
201
|
|
197
202
|
def self.logger
|
198
|
-
@logger ||= Sidekiq::Logger.new(
|
203
|
+
@logger ||= Sidekiq::Logger.new($stdout, level: Logger::INFO)
|
199
204
|
end
|
200
205
|
|
201
206
|
def self.logger=(logger)
|
207
|
+
if logger.nil?
|
208
|
+
self.logger.level = Logger::FATAL
|
209
|
+
return self.logger
|
210
|
+
end
|
211
|
+
|
212
|
+
logger.extend(Sidekiq::LoggingUtils)
|
213
|
+
|
202
214
|
@logger = logger
|
203
215
|
end
|
204
216
|
|
217
|
+
def self.pro?
|
218
|
+
defined?(Sidekiq::Pro)
|
219
|
+
end
|
220
|
+
|
205
221
|
# How frequently Redis should be checked by a random Sidekiq process for
|
206
222
|
# scheduled and retriable jobs. Each individual process will take turns by
|
207
223
|
# waiting some multiple of this value.
|
data/sidekiq.gemspec
CHANGED
@@ -5,17 +5,24 @@ Gem::Specification.new do |gem|
|
|
5
5
|
gem.email = ["mperham@gmail.com"]
|
6
6
|
gem.summary = "Simple, efficient background processing for Ruby"
|
7
7
|
gem.description = "Simple, efficient background processing for Ruby."
|
8
|
-
gem.homepage = "
|
8
|
+
gem.homepage = "https://sidekiq.org"
|
9
9
|
gem.license = "LGPL-3.0"
|
10
10
|
|
11
|
-
gem.executables = ["sidekiq"]
|
12
|
-
gem.files = `git ls-files | grep -
|
11
|
+
gem.executables = ["sidekiq", "sidekiqmon"]
|
12
|
+
gem.files = ["sidekiq.gemspec", "README.md", "Changes.md", "LICENSE"] + `git ls-files | grep -E '^(bin|lib|web)'`.split("\n")
|
13
13
|
gem.name = "sidekiq"
|
14
14
|
gem.version = Sidekiq::VERSION
|
15
15
|
gem.required_ruby_version = ">= 2.5.0"
|
16
16
|
|
17
|
-
gem.
|
17
|
+
gem.metadata = {
|
18
|
+
"homepage_uri" => "https://sidekiq.org",
|
19
|
+
"bug_tracker_uri" => "https://github.com/mperham/sidekiq/issues",
|
20
|
+
"documentation_uri" => "https://github.com/mperham/sidekiq/wiki",
|
21
|
+
"changelog_uri" => "https://github.com/mperham/sidekiq/blob/main/Changes.md",
|
22
|
+
"source_code_uri" => "https://github.com/mperham/sidekiq"
|
23
|
+
}
|
24
|
+
|
25
|
+
gem.add_dependency "redis", ">= 4.2.0"
|
18
26
|
gem.add_dependency "connection_pool", ">= 2.2.2"
|
19
|
-
gem.add_dependency "rack", "
|
20
|
-
gem.add_dependency "rack-protection", ">= 2.0.0"
|
27
|
+
gem.add_dependency "rack", "~> 2.0"
|
21
28
|
end
|
Binary file
|