sidekiq 6.5.8 → 7.1.0
Sign up to get free protection for your applications and to get access to all the features.
Potentially problematic release.
This version of sidekiq might be problematic. Click here for more details.
- checksums.yaml +4 -4
- data/Changes.md +113 -15
- data/README.md +40 -32
- data/bin/sidekiq +3 -8
- data/bin/sidekiqload +204 -118
- data/bin/sidekiqmon +3 -0
- data/lib/sidekiq/api.rb +110 -124
- data/lib/sidekiq/capsule.rb +127 -0
- data/lib/sidekiq/cli.rb +56 -74
- data/lib/sidekiq/client.rb +61 -37
- data/lib/sidekiq/component.rb +4 -1
- data/lib/sidekiq/config.rb +274 -0
- data/lib/sidekiq/deploy.rb +62 -0
- data/lib/sidekiq/embedded.rb +61 -0
- data/lib/sidekiq/fetch.rb +11 -14
- data/lib/sidekiq/job.rb +371 -10
- data/lib/sidekiq/job_logger.rb +2 -2
- data/lib/sidekiq/job_retry.rb +17 -14
- data/lib/sidekiq/job_util.rb +49 -15
- data/lib/sidekiq/launcher.rb +66 -62
- data/lib/sidekiq/logger.rb +1 -26
- data/lib/sidekiq/manager.rb +9 -11
- data/lib/sidekiq/metrics/query.rb +4 -4
- data/lib/sidekiq/metrics/shared.rb +7 -6
- data/lib/sidekiq/metrics/tracking.rb +20 -18
- data/lib/sidekiq/middleware/chain.rb +19 -18
- data/lib/sidekiq/middleware/current_attributes.rb +8 -15
- data/lib/sidekiq/monitor.rb +17 -4
- data/lib/sidekiq/paginator.rb +3 -3
- data/lib/sidekiq/processor.rb +21 -27
- data/lib/sidekiq/rails.rb +8 -7
- data/lib/sidekiq/redis_client_adapter.rb +11 -69
- data/lib/sidekiq/redis_connection.rb +11 -111
- data/lib/sidekiq/scheduled.rb +21 -22
- data/lib/sidekiq/testing.rb +5 -33
- data/lib/sidekiq/transaction_aware_client.rb +4 -5
- data/lib/sidekiq/version.rb +2 -1
- data/lib/sidekiq/web/application.rb +21 -6
- data/lib/sidekiq/web/csrf_protection.rb +1 -1
- data/lib/sidekiq/web/helpers.rb +19 -18
- data/lib/sidekiq/web.rb +6 -17
- data/lib/sidekiq/worker_compatibility_alias.rb +13 -0
- data/lib/sidekiq.rb +76 -274
- data/sidekiq.gemspec +12 -10
- data/web/assets/javascripts/application.js +18 -0
- data/web/assets/javascripts/base-charts.js +106 -0
- data/web/assets/javascripts/dashboard-charts.js +166 -0
- data/web/assets/javascripts/dashboard.js +3 -223
- data/web/assets/javascripts/metrics.js +117 -115
- data/web/assets/stylesheets/application-dark.css +4 -0
- data/web/assets/stylesheets/application-rtl.css +2 -91
- data/web/assets/stylesheets/application.css +23 -298
- data/web/locales/ar.yml +70 -70
- data/web/locales/cs.yml +62 -62
- data/web/locales/da.yml +60 -53
- data/web/locales/de.yml +65 -65
- data/web/locales/el.yml +2 -7
- data/web/locales/en.yml +76 -70
- data/web/locales/es.yml +68 -68
- data/web/locales/fa.yml +65 -65
- data/web/locales/fr.yml +81 -67
- data/web/locales/gd.yml +99 -0
- data/web/locales/he.yml +65 -64
- data/web/locales/hi.yml +59 -59
- data/web/locales/it.yml +53 -53
- data/web/locales/ja.yml +67 -69
- data/web/locales/ko.yml +52 -52
- data/web/locales/lt.yml +66 -66
- data/web/locales/nb.yml +61 -61
- data/web/locales/nl.yml +52 -52
- data/web/locales/pl.yml +45 -45
- data/web/locales/pt-br.yml +59 -69
- data/web/locales/pt.yml +51 -51
- data/web/locales/ru.yml +67 -66
- data/web/locales/sv.yml +53 -53
- data/web/locales/ta.yml +60 -60
- data/web/locales/uk.yml +62 -61
- data/web/locales/ur.yml +64 -64
- data/web/locales/vi.yml +67 -67
- data/web/locales/zh-cn.yml +20 -18
- data/web/locales/zh-tw.yml +10 -1
- data/web/views/_footer.erb +5 -2
- data/web/views/_job_info.erb +18 -2
- data/web/views/_metrics_period_select.erb +12 -0
- data/web/views/_paging.erb +2 -0
- data/web/views/_poll_link.erb +1 -1
- data/web/views/busy.erb +39 -28
- data/web/views/dashboard.erb +36 -5
- data/web/views/metrics.erb +33 -20
- data/web/views/metrics_for_job.erb +25 -44
- data/web/views/morgue.erb +5 -9
- data/web/views/queue.erb +10 -14
- data/web/views/queues.erb +3 -1
- data/web/views/retries.erb +5 -9
- data/web/views/scheduled.erb +12 -13
- metadata +43 -39
- data/lib/sidekiq/delay.rb +0 -43
- data/lib/sidekiq/extensions/action_mailer.rb +0 -48
- data/lib/sidekiq/extensions/active_record.rb +0 -43
- data/lib/sidekiq/extensions/class_methods.rb +0 -43
- data/lib/sidekiq/extensions/generic_proxy.rb +0 -33
- data/lib/sidekiq/metrics/deploy.rb +0 -47
- data/lib/sidekiq/worker.rb +0 -370
- data/web/assets/javascripts/graph.js +0 -16
- /data/{LICENSE → LICENSE.txt} +0 -0
data/lib/sidekiq/cli.rb
CHANGED
@@ -9,20 +9,11 @@ require "erb"
|
|
9
9
|
require "fileutils"
|
10
10
|
|
11
11
|
require "sidekiq"
|
12
|
+
require "sidekiq/config"
|
12
13
|
require "sidekiq/component"
|
14
|
+
require "sidekiq/capsule"
|
13
15
|
require "sidekiq/launcher"
|
14
16
|
|
15
|
-
# module ScoutApm
|
16
|
-
# VERSION = "5.3.1"
|
17
|
-
# end
|
18
|
-
fail <<~EOM if defined?(ScoutApm::VERSION) && ScoutApm::VERSION < "5.2.0"
|
19
|
-
|
20
|
-
|
21
|
-
scout_apm v#{ScoutApm::VERSION} is unsafe with Sidekiq 6.5. Please run `bundle up scout_apm` to upgrade to 5.2.0 or greater.
|
22
|
-
|
23
|
-
|
24
|
-
EOM
|
25
|
-
|
26
17
|
module Sidekiq # :nodoc:
|
27
18
|
class CLI
|
28
19
|
include Sidekiq::Component
|
@@ -33,9 +24,7 @@ module Sidekiq # :nodoc:
|
|
33
24
|
attr_accessor :config
|
34
25
|
|
35
26
|
def parse(args = ARGV.dup)
|
36
|
-
@config
|
37
|
-
@config[:error_handlers].clear
|
38
|
-
@config[:error_handlers] << @config.method(:default_error_handler)
|
27
|
+
@config ||= Sidekiq.default_configuration
|
39
28
|
|
40
29
|
setup_options(args)
|
41
30
|
initialize_logger
|
@@ -52,7 +41,7 @@ module Sidekiq # :nodoc:
|
|
52
41
|
def run(boot_app: true)
|
53
42
|
boot_application if boot_app
|
54
43
|
|
55
|
-
if environment == "development" && $stdout.tty? && @config.
|
44
|
+
if environment == "development" && $stdout.tty? && @config.logger.formatter.is_a?(Sidekiq::Logger::Formatters::Pretty)
|
56
45
|
print_banner
|
57
46
|
end
|
58
47
|
logger.info "Booted Rails #{::Rails.version} application in #{environment} environment" if rails_app?
|
@@ -84,26 +73,27 @@ module Sidekiq # :nodoc:
|
|
84
73
|
# touch the connection pool so it is created before we
|
85
74
|
# fire startup and start multithreading.
|
86
75
|
info = @config.redis_info
|
87
|
-
ver = info["redis_version"]
|
88
|
-
raise "You are connecting to Redis
|
76
|
+
ver = Gem::Version.new(info["redis_version"])
|
77
|
+
raise "You are connecting to Redis #{ver}, Sidekiq requires Redis 6.2.0 or greater" if ver < Gem::Version.new("6.2.0")
|
89
78
|
|
90
79
|
maxmemory_policy = info["maxmemory_policy"]
|
91
|
-
if maxmemory_policy != "noeviction"
|
80
|
+
if maxmemory_policy != "noeviction" && maxmemory_policy != ""
|
81
|
+
# Redis Enterprise Cloud returns "" for their policy 😳
|
92
82
|
logger.warn <<~EOM
|
93
83
|
|
94
84
|
|
95
85
|
WARNING: Your Redis instance will evict Sidekiq data under heavy load.
|
96
86
|
The 'noeviction' maxmemory policy is recommended (current policy: '#{maxmemory_policy}').
|
97
|
-
See: https://github.com/
|
87
|
+
See: https://github.com/sidekiq/sidekiq/wiki/Using-Redis#memory
|
98
88
|
|
99
89
|
EOM
|
100
90
|
end
|
101
91
|
|
102
92
|
# Since the user can pass us a connection pool explicitly in the initializer, we
|
103
93
|
# need to verify the size is large enough or else Sidekiq's performance is dramatically slowed.
|
104
|
-
|
105
|
-
|
106
|
-
|
94
|
+
@config.capsules.each_pair do |name, cap|
|
95
|
+
raise ArgumentError, "Pool size too small for #{name}" if cap.redis_pool.size < cap.concurrency
|
96
|
+
end
|
107
97
|
|
108
98
|
# cache process identity
|
109
99
|
@config[:identity] = identity
|
@@ -115,8 +105,8 @@ module Sidekiq # :nodoc:
|
|
115
105
|
# Starting here the process will now have multiple threads running.
|
116
106
|
fire_event(:startup, reverse: false, reraise: true)
|
117
107
|
|
118
|
-
logger.debug { "Client Middleware: #{@config.client_middleware.map(&:klass).join(", ")}" }
|
119
|
-
logger.debug { "Server Middleware: #{@config.server_middleware.map(&:klass).join(", ")}" }
|
108
|
+
logger.debug { "Client Middleware: #{@config.default_capsule.client_middleware.map(&:klass).join(", ")}" }
|
109
|
+
logger.debug { "Server Middleware: #{@config.default_capsule.server_middleware.map(&:klass).join(", ")}" }
|
120
110
|
|
121
111
|
launch(self_read)
|
122
112
|
end
|
@@ -149,19 +139,34 @@ module Sidekiq # :nodoc:
|
|
149
139
|
end
|
150
140
|
end
|
151
141
|
|
152
|
-
|
153
|
-
|
142
|
+
HOLIDAY_COLORS = {
|
143
|
+
# got other color-specific holidays from around the world?
|
144
|
+
# https://developer-book.com/post/definitive-guide-for-colored-text-in-terminal/#256-color-escape-codes
|
145
|
+
"3-17" => "\e[1;32m", # St. Patrick's Day green
|
146
|
+
"10-31" => "\e[38;5;208m" # Halloween orange
|
147
|
+
}
|
148
|
+
|
149
|
+
def self.day
|
150
|
+
@@day ||= begin
|
151
|
+
t = Date.today
|
152
|
+
"#{t.month}-#{t.day}"
|
153
|
+
end
|
154
154
|
end
|
155
155
|
|
156
156
|
def self.r
|
157
|
-
"\e[31m"
|
157
|
+
@@r ||= HOLIDAY_COLORS[day] || "\e[1;31m"
|
158
158
|
end
|
159
159
|
|
160
160
|
def self.b
|
161
|
-
"\e[30m"
|
161
|
+
@@b ||= HOLIDAY_COLORS[day] || "\e[30m"
|
162
|
+
end
|
163
|
+
|
164
|
+
def self.w
|
165
|
+
"\e[1;37m"
|
162
166
|
end
|
163
167
|
|
164
168
|
def self.reset
|
169
|
+
@@b = @@r = @@day = nil
|
165
170
|
"\e[0m"
|
166
171
|
end
|
167
172
|
|
@@ -174,7 +179,7 @@ module Sidekiq # :nodoc:
|
|
174
179
|
#{w} ,$$$$$b#{b}/#{w}md$$$P^'
|
175
180
|
#{w} .d$$$$$$#{b}/#{w}$$$P'
|
176
181
|
#{w} $$^' `"#{b}/#{w}$$$' #{r}____ _ _ _ _
|
177
|
-
#{w} $:
|
182
|
+
#{w} $: #{b}'#{w},$$: #{r} / ___|(_) __| | ___| | _(_) __ _
|
178
183
|
#{w} `b :$$ #{r} \\___ \\| |/ _` |/ _ \\ |/ / |/ _` |
|
179
184
|
#{w} $$: #{r} ___) | | (_| | __/ <| | (_| |
|
180
185
|
#{w} $$ #{r}|____/|_|\\__,_|\\___|_|\\_\\_|\\__, |
|
@@ -271,6 +276,18 @@ module Sidekiq # :nodoc:
|
|
271
276
|
|
272
277
|
# merge with defaults
|
273
278
|
@config.merge!(opts)
|
279
|
+
|
280
|
+
@config.default_capsule.tap do |cap|
|
281
|
+
cap.queues = opts[:queues]
|
282
|
+
cap.concurrency = opts[:concurrency] || @config[:concurrency]
|
283
|
+
end
|
284
|
+
|
285
|
+
opts[:capsules]&.each do |name, cap_config|
|
286
|
+
@config.capsule(name.to_s) do |cap|
|
287
|
+
cap.queues = cap_config[:queues]
|
288
|
+
cap.concurrency = cap_config[:concurrency]
|
289
|
+
end
|
290
|
+
end
|
274
291
|
end
|
275
292
|
|
276
293
|
def boot_application
|
@@ -278,12 +295,11 @@ module Sidekiq # :nodoc:
|
|
278
295
|
|
279
296
|
if File.directory?(@config[:require])
|
280
297
|
require "rails"
|
281
|
-
if ::Rails::VERSION::MAJOR <
|
282
|
-
|
283
|
-
else
|
284
|
-
require "sidekiq/rails"
|
285
|
-
require File.expand_path("#{@config[:require]}/config/environment.rb")
|
298
|
+
if ::Rails::VERSION::MAJOR < 6
|
299
|
+
warn "Sidekiq #{Sidekiq::VERSION} only supports Rails 6+"
|
286
300
|
end
|
301
|
+
require "sidekiq/rails"
|
302
|
+
require File.expand_path("#{@config[:require]}/config/environment.rb")
|
287
303
|
@config[:tag] ||= default_tag
|
288
304
|
else
|
289
305
|
require @config[:require]
|
@@ -331,10 +347,6 @@ module Sidekiq # :nodoc:
|
|
331
347
|
opts[:concurrency] = Integer(arg)
|
332
348
|
end
|
333
349
|
|
334
|
-
o.on "-d", "--daemon", "Daemonize process" do |arg|
|
335
|
-
puts "ERROR: Daemonization mode was removed in Sidekiq 6.0, please use a proper process supervisor to start and manage your services"
|
336
|
-
end
|
337
|
-
|
338
350
|
o.on "-e", "--environment ENV", "Application environment" do |arg|
|
339
351
|
opts[:environment] = arg
|
340
352
|
end
|
@@ -344,8 +356,8 @@ module Sidekiq # :nodoc:
|
|
344
356
|
end
|
345
357
|
|
346
358
|
o.on "-q", "--queue QUEUE[,WEIGHT]", "Queues to process with optional weights" do |arg|
|
347
|
-
|
348
|
-
|
359
|
+
opts[:queues] ||= []
|
360
|
+
opts[:queues] << arg
|
349
361
|
end
|
350
362
|
|
351
363
|
o.on "-r", "--require [PATH|DIR]", "Location of Rails application with jobs or file to require" do |arg|
|
@@ -364,15 +376,7 @@ module Sidekiq # :nodoc:
|
|
364
376
|
opts[:config_file] = arg
|
365
377
|
end
|
366
378
|
|
367
|
-
o.on "-
|
368
|
-
puts "ERROR: Logfile redirection was removed in Sidekiq 6.0, Sidekiq will only log to STDOUT"
|
369
|
-
end
|
370
|
-
|
371
|
-
o.on "-P", "--pidfile PATH", "path to pidfile" do |arg|
|
372
|
-
puts "ERROR: PID file creation was removed in Sidekiq 6.0, please use a proper process supervisor to start and manage your services"
|
373
|
-
end
|
374
|
-
|
375
|
-
o.on "-V", "--version", "Print version and exit" do |arg|
|
379
|
+
o.on "-V", "--version", "Print version and exit" do
|
376
380
|
puts "Sidekiq #{Sidekiq::VERSION}"
|
377
381
|
die(0)
|
378
382
|
end
|
@@ -392,9 +396,9 @@ module Sidekiq # :nodoc:
|
|
392
396
|
end
|
393
397
|
|
394
398
|
def parse_config(path)
|
395
|
-
erb = ERB.new(File.read(path))
|
399
|
+
erb = ERB.new(File.read(path), trim_mode: "-")
|
396
400
|
erb.filename = File.expand_path(path)
|
397
|
-
opts =
|
401
|
+
opts = YAML.safe_load(erb.result, permitted_classes: [Symbol], aliases: true) || {}
|
398
402
|
|
399
403
|
if opts.respond_to? :deep_symbolize_keys!
|
400
404
|
opts.deep_symbolize_keys!
|
@@ -405,31 +409,9 @@ module Sidekiq # :nodoc:
|
|
405
409
|
opts = opts.merge(opts.delete(environment.to_sym) || {})
|
406
410
|
opts.delete(:strict)
|
407
411
|
|
408
|
-
parse_queues(opts, opts.delete(:queues) || [])
|
409
|
-
|
410
412
|
opts
|
411
413
|
end
|
412
414
|
|
413
|
-
def load_yaml(src)
|
414
|
-
if Psych::VERSION > "4.0"
|
415
|
-
YAML.safe_load(src, permitted_classes: [Symbol], aliases: true)
|
416
|
-
else
|
417
|
-
YAML.load(src)
|
418
|
-
end
|
419
|
-
end
|
420
|
-
|
421
|
-
def parse_queues(opts, queues_and_weights)
|
422
|
-
queues_and_weights.each { |queue_and_weight| parse_queue(opts, *queue_and_weight) }
|
423
|
-
end
|
424
|
-
|
425
|
-
def parse_queue(opts, queue, weight = nil)
|
426
|
-
opts[:queues] ||= []
|
427
|
-
opts[:strict] = true if opts[:strict].nil?
|
428
|
-
raise ArgumentError, "queues: #{queue} cannot be defined twice" if opts[:queues].include?(queue)
|
429
|
-
[weight.to_i, 1].max.times { opts[:queues] << queue.to_s }
|
430
|
-
opts[:strict] = false if weight.to_i > 0
|
431
|
-
end
|
432
|
-
|
433
415
|
def rails_app?
|
434
416
|
defined?(::Rails) && ::Rails.respond_to?(:application)
|
435
417
|
end
|
@@ -437,4 +419,4 @@ module Sidekiq # :nodoc:
|
|
437
419
|
end
|
438
420
|
|
439
421
|
require "sidekiq/systemd"
|
440
|
-
require "sidekiq/metrics/tracking"
|
422
|
+
require "sidekiq/metrics/tracking"
|
data/lib/sidekiq/client.rb
CHANGED
@@ -21,7 +21,6 @@ module Sidekiq
|
|
21
21
|
# Sidekiq.client_middleware but you can change as necessary.
|
22
22
|
#
|
23
23
|
def middleware(&block)
|
24
|
-
@chain ||= Sidekiq.client_middleware
|
25
24
|
if block
|
26
25
|
@chain = @chain.dup
|
27
26
|
yield @chain
|
@@ -31,18 +30,32 @@ module Sidekiq
|
|
31
30
|
|
32
31
|
attr_accessor :redis_pool
|
33
32
|
|
34
|
-
# Sidekiq::Client
|
35
|
-
#
|
36
|
-
# Sidekiq jobs across several Redis instances (for scalability
|
37
|
-
# reasons, e.g.)
|
33
|
+
# Sidekiq::Client is responsible for pushing job payloads to Redis.
|
34
|
+
# Requires the :pool or :config keyword argument.
|
38
35
|
#
|
39
|
-
# Sidekiq::Client.new(
|
36
|
+
# Sidekiq::Client.new(pool: Sidekiq::RedisConnection.create)
|
40
37
|
#
|
41
|
-
#
|
42
|
-
#
|
43
|
-
#
|
44
|
-
|
45
|
-
|
38
|
+
# Inside the Sidekiq process, you can reuse the configured resources:
|
39
|
+
#
|
40
|
+
# Sidekiq::Client.new(config: config)
|
41
|
+
#
|
42
|
+
# @param pool [ConnectionPool] explicit Redis pool to use
|
43
|
+
# @param config [Sidekiq::Config] use the pool and middleware from the given Sidekiq container
|
44
|
+
# @param chain [Sidekiq::Middleware::Chain] use the given middleware chain
|
45
|
+
def initialize(*args, **kwargs)
|
46
|
+
if args.size == 1 && kwargs.size == 0
|
47
|
+
warn "Sidekiq::Client.new(pool) is deprecated, please use Sidekiq::Client.new(pool: pool), #{caller(0..3)}"
|
48
|
+
# old calling method, accept 1 pool argument
|
49
|
+
@redis_pool = args[0]
|
50
|
+
@chain = Sidekiq.default_configuration.client_middleware
|
51
|
+
@config = Sidekiq.default_configuration
|
52
|
+
else
|
53
|
+
# new calling method: keyword arguments
|
54
|
+
@config = kwargs[:config] || Sidekiq.default_configuration
|
55
|
+
@redis_pool = kwargs[:pool] || Thread.current[:sidekiq_redis_pool] || @config&.redis_pool
|
56
|
+
@chain = kwargs[:chain] || @config&.client_middleware
|
57
|
+
raise ArgumentError, "No Redis pool available for Sidekiq::Client" unless @redis_pool
|
58
|
+
end
|
46
59
|
end
|
47
60
|
|
48
61
|
##
|
@@ -83,8 +96,9 @@ module Sidekiq
|
|
83
96
|
|
84
97
|
##
|
85
98
|
# Push a large number of jobs to Redis. This method cuts out the redis
|
86
|
-
# network round trip latency.
|
87
|
-
# 1000
|
99
|
+
# network round trip latency. It pushes jobs in batches if more than
|
100
|
+
# `:batch_size` (1000 by default) of jobs are passed. I wouldn't recommend making `:batch_size`
|
101
|
+
# larger than 1000 but YMMV based on network quality, size of job args, etc.
|
88
102
|
# A large number of jobs can cause a bit of Redis command processing latency.
|
89
103
|
#
|
90
104
|
# Takes the same arguments as #push except that args is expected to be
|
@@ -92,13 +106,15 @@ module Sidekiq
|
|
92
106
|
# is run through the client middleware pipeline and each job gets its own Job ID
|
93
107
|
# as normal.
|
94
108
|
#
|
95
|
-
# Returns an array of the of pushed jobs' jids
|
96
|
-
#
|
109
|
+
# Returns an array of the of pushed jobs' jids, may contain nils if any client middleware
|
110
|
+
# prevented a job push.
|
111
|
+
#
|
112
|
+
# Example (pushing jobs in batches):
|
113
|
+
# push_bulk('class' => 'MyJob', 'args' => (1..100_000).to_a, batch_size: 1_000)
|
114
|
+
#
|
97
115
|
def push_bulk(items)
|
116
|
+
batch_size = items.delete(:batch_size) || items.delete("batch_size") || 1_000
|
98
117
|
args = items["args"]
|
99
|
-
raise ArgumentError, "Bulk arguments must be an Array of Arrays: [[1], [2]]" unless args.is_a?(Array) && args.all?(Array)
|
100
|
-
return [] if args.empty? # no jobs to push
|
101
|
-
|
102
118
|
at = items.delete("at")
|
103
119
|
raise ArgumentError, "Job 'at' must be a Numeric or an Array of Numeric timestamps" if at && (Array(at).empty? || !Array(at).all? { |entry| entry.is_a?(Numeric) })
|
104
120
|
raise ArgumentError, "Job 'at' Array must have same size as 'args' Array" if at.is_a?(Array) && at.size != args.size
|
@@ -107,18 +123,26 @@ module Sidekiq
|
|
107
123
|
raise ArgumentError, "Explicitly passing 'jid' when pushing more than one job is not supported" if jid && args.size > 1
|
108
124
|
|
109
125
|
normed = normalize_item(items)
|
110
|
-
|
111
|
-
|
112
|
-
|
113
|
-
|
114
|
-
|
115
|
-
copy
|
116
|
-
|
117
|
-
|
118
|
-
|
126
|
+
result = args.each_slice(batch_size).flat_map do |slice|
|
127
|
+
raise ArgumentError, "Bulk arguments must be an Array of Arrays: [[1], [2]]" unless slice.is_a?(Array) && slice.all?(Array)
|
128
|
+
break [] if slice.empty? # no jobs to push
|
129
|
+
|
130
|
+
payloads = slice.map.with_index { |job_args, index|
|
131
|
+
copy = normed.merge("args" => job_args, "jid" => SecureRandom.hex(12))
|
132
|
+
copy["at"] = (at.is_a?(Array) ? at[index] : at) if at
|
133
|
+
result = middleware.invoke(items["class"], copy, copy["queue"], @redis_pool) do
|
134
|
+
verify_json(copy)
|
135
|
+
copy
|
136
|
+
end
|
137
|
+
result || nil
|
138
|
+
}
|
139
|
+
|
140
|
+
to_push = payloads.compact
|
141
|
+
raw_push(to_push) unless to_push.empty?
|
142
|
+
payloads.map { |payload| payload&.[]("jid") }
|
143
|
+
end
|
119
144
|
|
120
|
-
|
121
|
-
payloads.collect { |payload| payload["jid"] }
|
145
|
+
result.is_a?(Enumerator::Lazy) ? result.force : result
|
122
146
|
end
|
123
147
|
|
124
148
|
# Allows sharding of jobs across any number of Redis instances. All jobs
|
@@ -135,11 +159,11 @@ module Sidekiq
|
|
135
159
|
# you cannot scale any other way (e.g. splitting your app into smaller apps).
|
136
160
|
def self.via(pool)
|
137
161
|
raise ArgumentError, "No pool given" if pool.nil?
|
138
|
-
current_sidekiq_pool = Thread.current[:
|
139
|
-
Thread.current[:
|
162
|
+
current_sidekiq_pool = Thread.current[:sidekiq_redis_pool]
|
163
|
+
Thread.current[:sidekiq_redis_pool] = pool
|
140
164
|
yield
|
141
165
|
ensure
|
142
|
-
Thread.current[:
|
166
|
+
Thread.current[:sidekiq_redis_pool] = current_sidekiq_pool
|
143
167
|
end
|
144
168
|
|
145
169
|
class << self
|
@@ -147,8 +171,8 @@ module Sidekiq
|
|
147
171
|
new.push(item)
|
148
172
|
end
|
149
173
|
|
150
|
-
def push_bulk(
|
151
|
-
new.push_bulk(
|
174
|
+
def push_bulk(...)
|
175
|
+
new.push_bulk(...)
|
152
176
|
end
|
153
177
|
|
154
178
|
# Resque compatibility helpers. Note all helpers
|
@@ -176,7 +200,7 @@ module Sidekiq
|
|
176
200
|
def enqueue_to_in(queue, interval, klass, *args)
|
177
201
|
int = interval.to_f
|
178
202
|
now = Time.now.to_f
|
179
|
-
ts = (int < 1_000_000_000 ? now + int : int)
|
203
|
+
ts = ((int < 1_000_000_000) ? now + int : int)
|
180
204
|
|
181
205
|
item = {"class" => klass, "args" => args, "at" => ts, "queue" => queue}
|
182
206
|
item.delete("at") if ts <= now
|
@@ -201,14 +225,14 @@ module Sidekiq
|
|
201
225
|
conn.pipelined do |pipeline|
|
202
226
|
atomic_push(pipeline, payloads)
|
203
227
|
end
|
204
|
-
rescue
|
228
|
+
rescue RedisClient::Error => ex
|
205
229
|
# 2550 Failover can cause the server to become a replica, need
|
206
230
|
# to disconnect and reopen the socket to get back to the primary.
|
207
231
|
# 4495 Use the same logic if we have a "Not enough replicas" error from the primary
|
208
232
|
# 4985 Use the same logic when a blocking command is force-unblocked
|
209
233
|
# The retry logic is copied from sidekiq.rb
|
210
234
|
if retryable && ex.message =~ /READONLY|NOREPLICAS|UNBLOCKED/
|
211
|
-
conn.
|
235
|
+
conn.close
|
212
236
|
retryable = false
|
213
237
|
retry
|
214
238
|
end
|
data/lib/sidekiq/component.rb
CHANGED
@@ -1,3 +1,5 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
module Sidekiq
|
2
4
|
##
|
3
5
|
# Sidekiq::Component assumes a config instance is available at @config
|
@@ -13,7 +15,7 @@ module Sidekiq
|
|
13
15
|
|
14
16
|
def safe_thread(name, &block)
|
15
17
|
Thread.new do
|
16
|
-
Thread.current.name = name
|
18
|
+
Thread.current.name = "sidekiq.#{name}"
|
17
19
|
watchdog(name, &block)
|
18
20
|
end
|
19
21
|
end
|
@@ -50,6 +52,7 @@ module Sidekiq
|
|
50
52
|
oneshot = options.fetch(:oneshot, true)
|
51
53
|
reverse = options[:reverse]
|
52
54
|
reraise = options[:reraise]
|
55
|
+
logger.debug("Firing #{event} event") if oneshot
|
53
56
|
|
54
57
|
arr = config[:lifecycle_events][event]
|
55
58
|
arr.reverse! if reverse
|