sidekiq 6.4.1 → 7.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of sidekiq might be problematic. Click here for more details.
- checksums.yaml +4 -4
- data/Changes.md +107 -5
- data/README.md +14 -13
- data/bin/sidekiq +3 -8
- data/bin/sidekiqload +26 -29
- data/lib/sidekiq/api.rb +232 -157
- data/lib/sidekiq/capsule.rb +110 -0
- data/lib/sidekiq/cli.rb +80 -86
- data/lib/sidekiq/client.rb +54 -42
- data/lib/sidekiq/component.rb +66 -0
- data/lib/sidekiq/config.rb +271 -0
- data/lib/sidekiq/deploy.rb +62 -0
- data/lib/sidekiq/embedded.rb +61 -0
- data/lib/sidekiq/fetch.rb +20 -19
- data/lib/sidekiq/job.rb +375 -10
- data/lib/sidekiq/job_logger.rb +1 -1
- data/lib/sidekiq/job_retry.rb +74 -53
- data/lib/sidekiq/job_util.rb +17 -11
- data/lib/sidekiq/launcher.rb +63 -69
- data/lib/sidekiq/logger.rb +6 -45
- data/lib/sidekiq/manager.rb +33 -32
- data/lib/sidekiq/metrics/query.rb +153 -0
- data/lib/sidekiq/metrics/shared.rb +95 -0
- data/lib/sidekiq/metrics/tracking.rb +134 -0
- data/lib/sidekiq/middleware/chain.rb +84 -42
- data/lib/sidekiq/middleware/current_attributes.rb +18 -17
- data/lib/sidekiq/middleware/i18n.rb +6 -4
- data/lib/sidekiq/middleware/modules.rb +21 -0
- data/lib/sidekiq/monitor.rb +1 -1
- data/lib/sidekiq/paginator.rb +10 -2
- data/lib/sidekiq/processor.rb +56 -59
- data/lib/sidekiq/rails.rb +10 -9
- data/lib/sidekiq/redis_client_adapter.rb +118 -0
- data/lib/sidekiq/redis_connection.rb +13 -82
- data/lib/sidekiq/ring_buffer.rb +29 -0
- data/lib/sidekiq/scheduled.rb +65 -37
- data/lib/sidekiq/testing/inline.rb +4 -4
- data/lib/sidekiq/testing.rb +41 -68
- data/lib/sidekiq/transaction_aware_client.rb +44 -0
- data/lib/sidekiq/version.rb +2 -1
- data/lib/sidekiq/web/action.rb +3 -3
- data/lib/sidekiq/web/application.rb +22 -6
- data/lib/sidekiq/web/csrf_protection.rb +3 -3
- data/lib/sidekiq/web/helpers.rb +21 -19
- data/lib/sidekiq/web.rb +3 -14
- data/lib/sidekiq/worker_compatibility_alias.rb +13 -0
- data/lib/sidekiq.rb +84 -207
- data/sidekiq.gemspec +29 -5
- data/web/assets/javascripts/application.js +58 -26
- data/web/assets/javascripts/base-charts.js +106 -0
- data/web/assets/javascripts/chart.min.js +13 -0
- data/web/assets/javascripts/chartjs-plugin-annotation.min.js +7 -0
- data/web/assets/javascripts/dashboard-charts.js +166 -0
- data/web/assets/javascripts/dashboard.js +3 -240
- data/web/assets/javascripts/metrics.js +236 -0
- data/web/assets/stylesheets/application-rtl.css +2 -91
- data/web/assets/stylesheets/application.css +64 -297
- data/web/locales/ar.yml +70 -70
- data/web/locales/cs.yml +62 -62
- data/web/locales/da.yml +52 -52
- data/web/locales/de.yml +65 -65
- data/web/locales/el.yml +43 -24
- data/web/locales/en.yml +82 -69
- data/web/locales/es.yml +68 -68
- data/web/locales/fa.yml +65 -65
- data/web/locales/fr.yml +67 -67
- data/web/locales/he.yml +65 -64
- data/web/locales/hi.yml +59 -59
- data/web/locales/it.yml +53 -53
- data/web/locales/ja.yml +71 -68
- data/web/locales/ko.yml +52 -52
- data/web/locales/lt.yml +66 -66
- data/web/locales/nb.yml +61 -61
- data/web/locales/nl.yml +52 -52
- data/web/locales/pl.yml +45 -45
- data/web/locales/pt-br.yml +63 -55
- data/web/locales/pt.yml +51 -51
- data/web/locales/ru.yml +67 -66
- data/web/locales/sv.yml +53 -53
- data/web/locales/ta.yml +60 -60
- data/web/locales/uk.yml +62 -61
- data/web/locales/ur.yml +64 -64
- data/web/locales/vi.yml +67 -67
- data/web/locales/zh-cn.yml +37 -11
- data/web/locales/zh-tw.yml +42 -8
- data/web/views/_footer.erb +5 -2
- data/web/views/_nav.erb +1 -1
- data/web/views/_summary.erb +1 -1
- data/web/views/busy.erb +9 -4
- data/web/views/dashboard.erb +36 -4
- data/web/views/metrics.erb +80 -0
- data/web/views/metrics_for_job.erb +69 -0
- data/web/views/queue.erb +5 -1
- metadata +69 -22
- data/lib/sidekiq/delay.rb +0 -43
- data/lib/sidekiq/exception_handler.rb +0 -27
- data/lib/sidekiq/extensions/action_mailer.rb +0 -48
- data/lib/sidekiq/extensions/active_record.rb +0 -43
- data/lib/sidekiq/extensions/class_methods.rb +0 -43
- data/lib/sidekiq/extensions/generic_proxy.rb +0 -33
- data/lib/sidekiq/util.rb +0 -108
- data/lib/sidekiq/worker.rb +0 -362
- /data/{LICENSE → LICENSE.txt} +0 -0
@@ -0,0 +1,110 @@
|
|
1
|
+
require "sidekiq/component"
|
2
|
+
|
3
|
+
module Sidekiq
|
4
|
+
# A Sidekiq::Capsule is the set of resources necessary to
|
5
|
+
# process one or more queues with a given concurrency.
|
6
|
+
# One "default" Capsule is started but the user may declare additional
|
7
|
+
# Capsules in their initializer.
|
8
|
+
#
|
9
|
+
# This capsule will pull jobs from the "single" queue and process
|
10
|
+
# the jobs with one thread, meaning the jobs will be processed serially.
|
11
|
+
#
|
12
|
+
# Sidekiq.configure_server do |config|
|
13
|
+
# config.capsule("single-threaded") do |cap|
|
14
|
+
# cap.concurrency = 1
|
15
|
+
# cap.queues = %w(single)
|
16
|
+
# end
|
17
|
+
# end
|
18
|
+
class Capsule
|
19
|
+
include Sidekiq::Component
|
20
|
+
|
21
|
+
attr_reader :name
|
22
|
+
attr_reader :queues
|
23
|
+
attr_accessor :concurrency
|
24
|
+
|
25
|
+
def initialize(name, config)
|
26
|
+
@name = name
|
27
|
+
@config = config
|
28
|
+
@queues = ["default"]
|
29
|
+
@concurrency = config[:concurrency]
|
30
|
+
end
|
31
|
+
|
32
|
+
def fetcher
|
33
|
+
@fetcher ||= begin
|
34
|
+
inst = (config[:fetch_class] || Sidekiq::BasicFetch).new(self)
|
35
|
+
inst.setup(config[:fetch_setup]) if inst.respond_to?(:setup)
|
36
|
+
inst
|
37
|
+
end
|
38
|
+
end
|
39
|
+
|
40
|
+
def stop
|
41
|
+
fetcher&.bulk_requeue([])
|
42
|
+
end
|
43
|
+
|
44
|
+
def queues=(val)
|
45
|
+
@queues = Array(val).each_with_object([]) do |qstr, memo|
|
46
|
+
arr = qstr
|
47
|
+
arr = qstr.split(",") if qstr.is_a?(String)
|
48
|
+
name, weight = arr
|
49
|
+
[weight.to_i, 1].max.times do
|
50
|
+
memo << name
|
51
|
+
end
|
52
|
+
end
|
53
|
+
end
|
54
|
+
|
55
|
+
# Allow the middleware to be different per-capsule.
|
56
|
+
# Avoid if possible and add middleware globally so all
|
57
|
+
# capsules share the same chains. Easier to debug that way.
|
58
|
+
def client_middleware
|
59
|
+
@client_chain ||= config.client_middleware.copy_for(self)
|
60
|
+
yield @client_chain if block_given?
|
61
|
+
@client_chain
|
62
|
+
end
|
63
|
+
|
64
|
+
def server_middleware
|
65
|
+
@server_chain ||= config.server_middleware.copy_for(self)
|
66
|
+
yield @server_chain if block_given?
|
67
|
+
@server_chain
|
68
|
+
end
|
69
|
+
|
70
|
+
def redis_pool
|
71
|
+
Thread.current[:sidekiq_redis_pool] || local_redis_pool
|
72
|
+
end
|
73
|
+
|
74
|
+
def local_redis_pool
|
75
|
+
# connection pool is lazy, it will not create connections unless you actually need them
|
76
|
+
# so don't be skimpy!
|
77
|
+
@redis ||= config.new_redis_pool(@concurrency, name)
|
78
|
+
end
|
79
|
+
|
80
|
+
def redis
|
81
|
+
raise ArgumentError, "requires a block" unless block_given?
|
82
|
+
redis_pool.with do |conn|
|
83
|
+
retryable = true
|
84
|
+
begin
|
85
|
+
yield conn
|
86
|
+
rescue RedisClientAdapter::BaseError => ex
|
87
|
+
# 2550 Failover can cause the server to become a replica, need
|
88
|
+
# to disconnect and reopen the socket to get back to the primary.
|
89
|
+
# 4495 Use the same logic if we have a "Not enough replicas" error from the primary
|
90
|
+
# 4985 Use the same logic when a blocking command is force-unblocked
|
91
|
+
# The same retry logic is also used in client.rb
|
92
|
+
if retryable && ex.message =~ /READONLY|NOREPLICAS|UNBLOCKED/
|
93
|
+
conn.close
|
94
|
+
retryable = false
|
95
|
+
retry
|
96
|
+
end
|
97
|
+
raise
|
98
|
+
end
|
99
|
+
end
|
100
|
+
end
|
101
|
+
|
102
|
+
def lookup(name)
|
103
|
+
config.lookup(name)
|
104
|
+
end
|
105
|
+
|
106
|
+
def logger
|
107
|
+
config.logger
|
108
|
+
end
|
109
|
+
end
|
110
|
+
end
|
data/lib/sidekiq/cli.rb
CHANGED
@@ -9,18 +9,23 @@ require "erb"
|
|
9
9
|
require "fileutils"
|
10
10
|
|
11
11
|
require "sidekiq"
|
12
|
+
require "sidekiq/config"
|
13
|
+
require "sidekiq/component"
|
14
|
+
require "sidekiq/capsule"
|
12
15
|
require "sidekiq/launcher"
|
13
|
-
require "sidekiq/util"
|
14
16
|
|
15
|
-
module Sidekiq
|
17
|
+
module Sidekiq # :nodoc:
|
16
18
|
class CLI
|
17
|
-
include
|
19
|
+
include Sidekiq::Component
|
18
20
|
include Singleton unless $TESTING
|
19
21
|
|
20
22
|
attr_accessor :launcher
|
21
23
|
attr_accessor :environment
|
24
|
+
attr_accessor :config
|
25
|
+
|
26
|
+
def parse(args = ARGV.dup)
|
27
|
+
@config ||= Sidekiq.default_configuration
|
22
28
|
|
23
|
-
def parse(args = ARGV)
|
24
29
|
setup_options(args)
|
25
30
|
initialize_logger
|
26
31
|
validate!
|
@@ -36,7 +41,7 @@ module Sidekiq
|
|
36
41
|
def run(boot_app: true)
|
37
42
|
boot_application if boot_app
|
38
43
|
|
39
|
-
if environment == "development" && $stdout.tty? &&
|
44
|
+
if environment == "development" && $stdout.tty? && @config.logger.formatter.is_a?(Sidekiq::Logger::Formatters::Pretty)
|
40
45
|
print_banner
|
41
46
|
end
|
42
47
|
logger.info "Booted Rails #{::Rails.version} application in #{environment} environment" if rails_app?
|
@@ -67,9 +72,9 @@ module Sidekiq
|
|
67
72
|
|
68
73
|
# touch the connection pool so it is created before we
|
69
74
|
# fire startup and start multithreading.
|
70
|
-
info =
|
71
|
-
ver = info["redis_version"]
|
72
|
-
raise "You are connecting to Redis
|
75
|
+
info = @config.redis_info
|
76
|
+
ver = Gem::Version.new(info["redis_version"])
|
77
|
+
raise "You are connecting to Redis #{ver}, Sidekiq requires Redis 6.2.0 or greater" if ver < Gem::Version.new("6.2.0")
|
73
78
|
|
74
79
|
maxmemory_policy = info["maxmemory_policy"]
|
75
80
|
if maxmemory_policy != "noeviction"
|
@@ -85,22 +90,22 @@ module Sidekiq
|
|
85
90
|
|
86
91
|
# Since the user can pass us a connection pool explicitly in the initializer, we
|
87
92
|
# need to verify the size is large enough or else Sidekiq's performance is dramatically slowed.
|
88
|
-
|
89
|
-
|
90
|
-
|
93
|
+
@config.capsules.each_pair do |name, cap|
|
94
|
+
raise ArgumentError, "Pool size too small for #{name}" if cap.redis_pool.size < cap.concurrency
|
95
|
+
end
|
91
96
|
|
92
97
|
# cache process identity
|
93
|
-
|
98
|
+
@config[:identity] = identity
|
94
99
|
|
95
100
|
# Touch middleware so it isn't lazy loaded by multiple threads, #3043
|
96
|
-
|
101
|
+
@config.server_middleware
|
97
102
|
|
98
103
|
# Before this point, the process is initializing with just the main thread.
|
99
104
|
# Starting here the process will now have multiple threads running.
|
100
105
|
fire_event(:startup, reverse: false, reraise: true)
|
101
106
|
|
102
|
-
logger.debug { "Client Middleware: #{
|
103
|
-
logger.debug { "Server Middleware: #{
|
107
|
+
logger.debug { "Client Middleware: #{@config.default_capsule.client_middleware.map(&:klass).join(", ")}" }
|
108
|
+
logger.debug { "Server Middleware: #{@config.default_capsule.server_middleware.map(&:klass).join(", ")}" }
|
104
109
|
|
105
110
|
launch(self_read)
|
106
111
|
end
|
@@ -110,13 +115,13 @@ module Sidekiq
|
|
110
115
|
logger.info "Starting processing, hit Ctrl-C to stop"
|
111
116
|
end
|
112
117
|
|
113
|
-
@launcher = Sidekiq::Launcher.new(
|
118
|
+
@launcher = Sidekiq::Launcher.new(@config)
|
114
119
|
|
115
120
|
begin
|
116
121
|
launcher.run
|
117
122
|
|
118
|
-
while
|
119
|
-
signal =
|
123
|
+
while self_read.wait_readable
|
124
|
+
signal = self_read.gets.strip
|
120
125
|
handle_signal(signal)
|
121
126
|
end
|
122
127
|
rescue Interrupt
|
@@ -133,19 +138,34 @@ module Sidekiq
|
|
133
138
|
end
|
134
139
|
end
|
135
140
|
|
136
|
-
|
137
|
-
|
141
|
+
HOLIDAY_COLORS = {
|
142
|
+
# got other color-specific holidays from around the world?
|
143
|
+
# https://developer-book.com/post/definitive-guide-for-colored-text-in-terminal/#256-color-escape-codes
|
144
|
+
"3-17" => "\e[1;32m", # St. Patrick's Day green
|
145
|
+
"10-31" => "\e[38;5;208m" # Halloween orange
|
146
|
+
}
|
147
|
+
|
148
|
+
def self.day
|
149
|
+
@@day ||= begin
|
150
|
+
t = Date.today
|
151
|
+
"#{t.month}-#{t.day}"
|
152
|
+
end
|
138
153
|
end
|
139
154
|
|
140
155
|
def self.r
|
141
|
-
"\e[31m"
|
156
|
+
@@r ||= HOLIDAY_COLORS[day] || "\e[1;31m"
|
142
157
|
end
|
143
158
|
|
144
159
|
def self.b
|
145
|
-
"\e[30m"
|
160
|
+
@@b ||= HOLIDAY_COLORS[day] || "\e[30m"
|
161
|
+
end
|
162
|
+
|
163
|
+
def self.w
|
164
|
+
"\e[1;37m"
|
146
165
|
end
|
147
166
|
|
148
167
|
def self.reset
|
168
|
+
@@b = @@r = @@day = nil
|
149
169
|
"\e[0m"
|
150
170
|
end
|
151
171
|
|
@@ -158,7 +178,7 @@ module Sidekiq
|
|
158
178
|
#{w} ,$$$$$b#{b}/#{w}md$$$P^'
|
159
179
|
#{w} .d$$$$$$#{b}/#{w}$$$P'
|
160
180
|
#{w} $$^' `"#{b}/#{w}$$$' #{r}____ _ _ _ _
|
161
|
-
#{w} $:
|
181
|
+
#{w} $: #{b}'#{w},$$: #{r} / ___|(_) __| | ___| | _(_) __ _
|
162
182
|
#{w} `b :$$ #{r} \\___ \\| |/ _` |/ _ \\ |/ / |/ _` |
|
163
183
|
#{w} $$: #{r} ___) | | (_| | __/ <| | (_| |
|
164
184
|
#{w} $$ #{r}|____/|_|\\__,_|\\___|_|\\_\\_|\\__, |
|
@@ -173,25 +193,25 @@ module Sidekiq
|
|
173
193
|
# Heroku sends TERM and then waits 30 seconds for process to exit.
|
174
194
|
"TERM" => ->(cli) { raise Interrupt },
|
175
195
|
"TSTP" => ->(cli) {
|
176
|
-
|
196
|
+
cli.logger.info "Received TSTP, no longer accepting new work"
|
177
197
|
cli.launcher.quiet
|
178
198
|
},
|
179
199
|
"TTIN" => ->(cli) {
|
180
200
|
Thread.list.each do |thread|
|
181
|
-
|
201
|
+
cli.logger.warn "Thread TID-#{(thread.object_id ^ ::Process.pid).to_s(36)} #{thread.name}"
|
182
202
|
if thread.backtrace
|
183
|
-
|
203
|
+
cli.logger.warn thread.backtrace.join("\n")
|
184
204
|
else
|
185
|
-
|
205
|
+
cli.logger.warn "<no backtrace available>"
|
186
206
|
end
|
187
207
|
end
|
188
208
|
}
|
189
209
|
}
|
190
|
-
UNHANDLED_SIGNAL_HANDLER = ->(cli) {
|
210
|
+
UNHANDLED_SIGNAL_HANDLER = ->(cli) { cli.logger.info "No signal handler registered, ignoring" }
|
191
211
|
SIGNAL_HANDLERS.default = UNHANDLED_SIGNAL_HANDLER
|
192
212
|
|
193
213
|
def handle_signal(sig)
|
194
|
-
|
214
|
+
logger.debug "Got #{sig} signal"
|
195
215
|
SIGNAL_HANDLERS[sig].call(self)
|
196
216
|
end
|
197
217
|
|
@@ -237,7 +257,7 @@ module Sidekiq
|
|
237
257
|
config_dir = if File.directory?(opts[:require].to_s)
|
238
258
|
File.join(opts[:require], "config")
|
239
259
|
else
|
240
|
-
File.join(
|
260
|
+
File.join(@config[:require], "config")
|
241
261
|
end
|
242
262
|
|
243
263
|
%w[sidekiq.yml sidekiq.yml.erb].each do |config_file|
|
@@ -254,27 +274,34 @@ module Sidekiq
|
|
254
274
|
opts[:concurrency] = Integer(ENV["RAILS_MAX_THREADS"]) if opts[:concurrency].nil? && ENV["RAILS_MAX_THREADS"]
|
255
275
|
|
256
276
|
# merge with defaults
|
257
|
-
|
258
|
-
|
277
|
+
@config.merge!(opts)
|
278
|
+
|
279
|
+
@config.default_capsule.tap do |cap|
|
280
|
+
cap.queues = opts[:queues]
|
281
|
+
cap.concurrency = opts[:concurrency] || @config[:concurrency]
|
282
|
+
end
|
259
283
|
|
260
|
-
|
261
|
-
|
284
|
+
opts[:capsules]&.each do |name, cap_config|
|
285
|
+
@config.capsule(name.to_s) do |cap|
|
286
|
+
cap.queues = cap_config[:queues]
|
287
|
+
cap.concurrency = cap_config[:concurrency]
|
288
|
+
end
|
289
|
+
end
|
262
290
|
end
|
263
291
|
|
264
292
|
def boot_application
|
265
293
|
ENV["RACK_ENV"] = ENV["RAILS_ENV"] = environment
|
266
294
|
|
267
|
-
if File.directory?(
|
295
|
+
if File.directory?(@config[:require])
|
268
296
|
require "rails"
|
269
|
-
if ::Rails::VERSION::MAJOR <
|
270
|
-
|
271
|
-
else
|
272
|
-
require "sidekiq/rails"
|
273
|
-
require File.expand_path("#{options[:require]}/config/environment.rb")
|
297
|
+
if ::Rails::VERSION::MAJOR < 6
|
298
|
+
warn "Sidekiq #{Sidekiq::VERSION} only supports Rails 6+"
|
274
299
|
end
|
275
|
-
|
300
|
+
require "sidekiq/rails"
|
301
|
+
require File.expand_path("#{@config[:require]}/config/environment.rb")
|
302
|
+
@config[:tag] ||= default_tag
|
276
303
|
else
|
277
|
-
require
|
304
|
+
require @config[:require]
|
278
305
|
end
|
279
306
|
end
|
280
307
|
|
@@ -291,18 +318,18 @@ module Sidekiq
|
|
291
318
|
end
|
292
319
|
|
293
320
|
def validate!
|
294
|
-
if !File.exist?(
|
295
|
-
(File.directory?(
|
321
|
+
if !File.exist?(@config[:require]) ||
|
322
|
+
(File.directory?(@config[:require]) && !File.exist?("#{@config[:require]}/config/application.rb"))
|
296
323
|
logger.info "=================================================================="
|
297
324
|
logger.info " Please point Sidekiq to a Rails application or a Ruby file "
|
298
|
-
logger.info " to load your
|
325
|
+
logger.info " to load your job classes with -r [DIR|FILE]."
|
299
326
|
logger.info "=================================================================="
|
300
327
|
logger.info @parser
|
301
328
|
die(1)
|
302
329
|
end
|
303
330
|
|
304
331
|
[:concurrency, :timeout].each do |opt|
|
305
|
-
raise ArgumentError, "#{opt}: #{
|
332
|
+
raise ArgumentError, "#{opt}: #{@config[opt]} is not a valid value" if @config[opt].to_i <= 0
|
306
333
|
end
|
307
334
|
end
|
308
335
|
|
@@ -319,10 +346,6 @@ module Sidekiq
|
|
319
346
|
opts[:concurrency] = Integer(arg)
|
320
347
|
end
|
321
348
|
|
322
|
-
o.on "-d", "--daemon", "Daemonize process" do |arg|
|
323
|
-
puts "ERROR: Daemonization mode was removed in Sidekiq 6.0, please use a proper process supervisor to start and manage your services"
|
324
|
-
end
|
325
|
-
|
326
349
|
o.on "-e", "--environment ENV", "Application environment" do |arg|
|
327
350
|
opts[:environment] = arg
|
328
351
|
end
|
@@ -332,11 +355,11 @@ module Sidekiq
|
|
332
355
|
end
|
333
356
|
|
334
357
|
o.on "-q", "--queue QUEUE[,WEIGHT]", "Queues to process with optional weights" do |arg|
|
335
|
-
|
336
|
-
|
358
|
+
opts[:queues] ||= []
|
359
|
+
opts[:queues] << arg
|
337
360
|
end
|
338
361
|
|
339
|
-
o.on "-r", "--require [PATH|DIR]", "Location of Rails application with
|
362
|
+
o.on "-r", "--require [PATH|DIR]", "Location of Rails application with jobs or file to require" do |arg|
|
340
363
|
opts[:require] = arg
|
341
364
|
end
|
342
365
|
|
@@ -352,15 +375,7 @@ module Sidekiq
|
|
352
375
|
opts[:config_file] = arg
|
353
376
|
end
|
354
377
|
|
355
|
-
o.on "-
|
356
|
-
puts "ERROR: Logfile redirection was removed in Sidekiq 6.0, Sidekiq will only log to STDOUT"
|
357
|
-
end
|
358
|
-
|
359
|
-
o.on "-P", "--pidfile PATH", "path to pidfile" do |arg|
|
360
|
-
puts "ERROR: PID file creation was removed in Sidekiq 6.0, please use a proper process supervisor to start and manage your services"
|
361
|
-
end
|
362
|
-
|
363
|
-
o.on "-V", "--version", "Print version and exit" do |arg|
|
378
|
+
o.on "-V", "--version", "Print version and exit" do
|
364
379
|
puts "Sidekiq #{Sidekiq::VERSION}"
|
365
380
|
die(0)
|
366
381
|
end
|
@@ -376,13 +391,13 @@ module Sidekiq
|
|
376
391
|
end
|
377
392
|
|
378
393
|
def initialize_logger
|
379
|
-
|
394
|
+
@config.logger.level = ::Logger::DEBUG if @config[:verbose]
|
380
395
|
end
|
381
396
|
|
382
397
|
def parse_config(path)
|
383
398
|
erb = ERB.new(File.read(path))
|
384
399
|
erb.filename = File.expand_path(path)
|
385
|
-
opts =
|
400
|
+
opts = YAML.safe_load(erb.result, permitted_classes: [Symbol], aliases: true) || {}
|
386
401
|
|
387
402
|
if opts.respond_to? :deep_symbolize_keys!
|
388
403
|
opts.deep_symbolize_keys!
|
@@ -393,31 +408,9 @@ module Sidekiq
|
|
393
408
|
opts = opts.merge(opts.delete(environment.to_sym) || {})
|
394
409
|
opts.delete(:strict)
|
395
410
|
|
396
|
-
parse_queues(opts, opts.delete(:queues) || [])
|
397
|
-
|
398
411
|
opts
|
399
412
|
end
|
400
413
|
|
401
|
-
def load_yaml(src)
|
402
|
-
if Psych::VERSION > "4.0"
|
403
|
-
YAML.safe_load(src, permitted_classes: [Symbol], aliases: true)
|
404
|
-
else
|
405
|
-
YAML.load(src)
|
406
|
-
end
|
407
|
-
end
|
408
|
-
|
409
|
-
def parse_queues(opts, queues_and_weights)
|
410
|
-
queues_and_weights.each { |queue_and_weight| parse_queue(opts, *queue_and_weight) }
|
411
|
-
end
|
412
|
-
|
413
|
-
def parse_queue(opts, queue, weight = nil)
|
414
|
-
opts[:queues] ||= []
|
415
|
-
opts[:strict] = true if opts[:strict].nil?
|
416
|
-
raise ArgumentError, "queues: #{queue} cannot be defined twice" if opts[:queues].include?(queue)
|
417
|
-
[weight.to_i, 1].max.times { opts[:queues] << queue.to_s }
|
418
|
-
opts[:strict] = false if weight.to_i > 0
|
419
|
-
end
|
420
|
-
|
421
414
|
def rails_app?
|
422
415
|
defined?(::Rails) && ::Rails.respond_to?(:application)
|
423
416
|
end
|
@@ -425,3 +418,4 @@ module Sidekiq
|
|
425
418
|
end
|
426
419
|
|
427
420
|
require "sidekiq/systemd"
|
421
|
+
require "sidekiq/metrics/tracking"
|
data/lib/sidekiq/client.rb
CHANGED
@@ -15,13 +15,12 @@ module Sidekiq
|
|
15
15
|
# client.middleware do |chain|
|
16
16
|
# chain.use MyClientMiddleware
|
17
17
|
# end
|
18
|
-
# client.push('class' => '
|
18
|
+
# client.push('class' => 'SomeJob', 'args' => [1,2,3])
|
19
19
|
#
|
20
20
|
# All client instances default to the globally-defined
|
21
21
|
# Sidekiq.client_middleware but you can change as necessary.
|
22
22
|
#
|
23
23
|
def middleware(&block)
|
24
|
-
@chain ||= Sidekiq.client_middleware
|
25
24
|
if block
|
26
25
|
@chain = @chain.dup
|
27
26
|
yield @chain
|
@@ -31,34 +30,48 @@ module Sidekiq
|
|
31
30
|
|
32
31
|
attr_accessor :redis_pool
|
33
32
|
|
34
|
-
# Sidekiq::Client
|
35
|
-
#
|
36
|
-
# Sidekiq jobs across several Redis instances (for scalability
|
37
|
-
# reasons, e.g.)
|
33
|
+
# Sidekiq::Client is responsible for pushing job payloads to Redis.
|
34
|
+
# Requires the :pool or :config keyword argument.
|
38
35
|
#
|
39
|
-
# Sidekiq::Client.new(
|
36
|
+
# Sidekiq::Client.new(pool: Sidekiq::RedisConnection.create)
|
40
37
|
#
|
41
|
-
#
|
42
|
-
#
|
43
|
-
#
|
44
|
-
|
45
|
-
|
38
|
+
# Inside the Sidekiq process, you can reuse the configured resources:
|
39
|
+
#
|
40
|
+
# Sidekiq::Client.new(config: config)
|
41
|
+
#
|
42
|
+
# @param pool [ConnectionPool] explicit Redis pool to use
|
43
|
+
# @param config [Sidekiq::Config] use the pool and middleware from the given Sidekiq container
|
44
|
+
# @param chain [Sidekiq::Middleware::Chain] use the given middleware chain
|
45
|
+
def initialize(*args, **kwargs)
|
46
|
+
if args.size == 1 && kwargs.size == 0
|
47
|
+
warn "Sidekiq::Client.new(pool) is deprecated, please use Sidekiq::Client.new(pool: pool), #{caller(0..3)}"
|
48
|
+
# old calling method, accept 1 pool argument
|
49
|
+
@redis_pool = args[0]
|
50
|
+
@chain = Sidekiq.default_configuration.client_middleware
|
51
|
+
@config = Sidekiq.default_configuration
|
52
|
+
else
|
53
|
+
# new calling method: keyword arguments
|
54
|
+
@config = kwargs[:config] || Sidekiq.default_configuration
|
55
|
+
@redis_pool = kwargs[:pool] || Thread.current[:sidekiq_redis_pool] || @config&.redis_pool
|
56
|
+
@chain = kwargs[:chain] || @config&.client_middleware
|
57
|
+
raise ArgumentError, "No Redis pool available for Sidekiq::Client" unless @redis_pool
|
58
|
+
end
|
46
59
|
end
|
47
60
|
|
48
61
|
##
|
49
62
|
# The main method used to push a job to Redis. Accepts a number of options:
|
50
63
|
#
|
51
64
|
# queue - the named queue to use, default 'default'
|
52
|
-
# class - the
|
65
|
+
# class - the job class to call, required
|
53
66
|
# args - an array of simple arguments to the perform method, must be JSON-serializable
|
54
67
|
# at - timestamp to schedule the job (optional), must be Numeric (e.g. Time.now.to_f)
|
55
68
|
# retry - whether to retry this job if it fails, default true or an integer number of retries
|
56
69
|
# backtrace - whether to save any error backtrace, default false
|
57
70
|
#
|
58
71
|
# If class is set to the class name, the jobs' options will be based on Sidekiq's default
|
59
|
-
#
|
72
|
+
# job options. Otherwise, they will be based on the job class's options.
|
60
73
|
#
|
61
|
-
# Any options valid for a
|
74
|
+
# Any options valid for a job class's sidekiq_options are also available here.
|
62
75
|
#
|
63
76
|
# All options must be strings, not symbols. NB: because we are serializing to JSON, all
|
64
77
|
# symbols in 'args' will be converted to strings. Note that +backtrace: true+ can take quite a bit of
|
@@ -67,13 +80,15 @@ module Sidekiq
|
|
67
80
|
# Returns a unique Job ID. If middleware stops the job, nil will be returned instead.
|
68
81
|
#
|
69
82
|
# Example:
|
70
|
-
# push('queue' => 'my_queue', 'class' =>
|
83
|
+
# push('queue' => 'my_queue', 'class' => MyJob, 'args' => ['foo', 1, :bat => 'bar'])
|
71
84
|
#
|
72
85
|
def push(item)
|
73
86
|
normed = normalize_item(item)
|
74
|
-
payload =
|
75
|
-
|
87
|
+
payload = middleware.invoke(item["class"], normed, normed["queue"], @redis_pool) do
|
88
|
+
normed
|
89
|
+
end
|
76
90
|
if payload
|
91
|
+
verify_json(payload)
|
77
92
|
raw_push([payload])
|
78
93
|
payload["jid"]
|
79
94
|
end
|
@@ -101,12 +116,17 @@ module Sidekiq
|
|
101
116
|
raise ArgumentError, "Job 'at' must be a Numeric or an Array of Numeric timestamps" if at && (Array(at).empty? || !Array(at).all? { |entry| entry.is_a?(Numeric) })
|
102
117
|
raise ArgumentError, "Job 'at' Array must have same size as 'args' Array" if at.is_a?(Array) && at.size != args.size
|
103
118
|
|
119
|
+
jid = items.delete("jid")
|
120
|
+
raise ArgumentError, "Explicitly passing 'jid' when pushing more than one job is not supported" if jid && args.size > 1
|
121
|
+
|
104
122
|
normed = normalize_item(items)
|
105
123
|
payloads = args.map.with_index { |job_args, index|
|
106
124
|
copy = normed.merge("args" => job_args, "jid" => SecureRandom.hex(12))
|
107
125
|
copy["at"] = (at.is_a?(Array) ? at[index] : at) if at
|
108
|
-
|
109
|
-
|
126
|
+
result = middleware.invoke(items["class"], copy, copy["queue"], @redis_pool) do
|
127
|
+
verify_json(copy)
|
128
|
+
copy
|
129
|
+
end
|
110
130
|
result || nil
|
111
131
|
}.compact
|
112
132
|
|
@@ -119,8 +139,8 @@ module Sidekiq
|
|
119
139
|
#
|
120
140
|
# pool = ConnectionPool.new { Redis.new }
|
121
141
|
# Sidekiq::Client.via(pool) do
|
122
|
-
#
|
123
|
-
#
|
142
|
+
# SomeJob.perform_async(1,2,3)
|
143
|
+
# SomeOtherJob.perform_async(1,2,3)
|
124
144
|
# end
|
125
145
|
#
|
126
146
|
# Generally this is only needed for very large Sidekiq installs processing
|
@@ -128,11 +148,11 @@ module Sidekiq
|
|
128
148
|
# you cannot scale any other way (e.g. splitting your app into smaller apps).
|
129
149
|
def self.via(pool)
|
130
150
|
raise ArgumentError, "No pool given" if pool.nil?
|
131
|
-
current_sidekiq_pool = Thread.current[:
|
132
|
-
Thread.current[:
|
151
|
+
current_sidekiq_pool = Thread.current[:sidekiq_redis_pool]
|
152
|
+
Thread.current[:sidekiq_redis_pool] = pool
|
133
153
|
yield
|
134
154
|
ensure
|
135
|
-
Thread.current[:
|
155
|
+
Thread.current[:sidekiq_redis_pool] = current_sidekiq_pool
|
136
156
|
end
|
137
157
|
|
138
158
|
class << self
|
@@ -145,10 +165,10 @@ module Sidekiq
|
|
145
165
|
end
|
146
166
|
|
147
167
|
# Resque compatibility helpers. Note all helpers
|
148
|
-
# should go through
|
168
|
+
# should go through Sidekiq::Job#client_push.
|
149
169
|
#
|
150
170
|
# Example usage:
|
151
|
-
# Sidekiq::Client.enqueue(
|
171
|
+
# Sidekiq::Client.enqueue(MyJob, 'foo', 1, :bat => 'bar')
|
152
172
|
#
|
153
173
|
# Messages are enqueued to the 'default' queue.
|
154
174
|
#
|
@@ -157,14 +177,14 @@ module Sidekiq
|
|
157
177
|
end
|
158
178
|
|
159
179
|
# Example usage:
|
160
|
-
# Sidekiq::Client.enqueue_to(:queue_name,
|
180
|
+
# Sidekiq::Client.enqueue_to(:queue_name, MyJob, 'foo', 1, :bat => 'bar')
|
161
181
|
#
|
162
182
|
def enqueue_to(queue, klass, *args)
|
163
183
|
klass.client_push("queue" => queue, "class" => klass, "args" => args)
|
164
184
|
end
|
165
185
|
|
166
186
|
# Example usage:
|
167
|
-
# Sidekiq::Client.enqueue_to_in(:queue_name, 3.minutes,
|
187
|
+
# Sidekiq::Client.enqueue_to_in(:queue_name, 3.minutes, MyJob, 'foo', 1, :bat => 'bar')
|
168
188
|
#
|
169
189
|
def enqueue_to_in(queue, interval, klass, *args)
|
170
190
|
int = interval.to_f
|
@@ -178,7 +198,7 @@ module Sidekiq
|
|
178
198
|
end
|
179
199
|
|
180
200
|
# Example usage:
|
181
|
-
# Sidekiq::Client.enqueue_in(3.minutes,
|
201
|
+
# Sidekiq::Client.enqueue_in(3.minutes, MyJob, 'foo', 1, :bat => 'bar')
|
182
202
|
#
|
183
203
|
def enqueue_in(interval, klass, *args)
|
184
204
|
klass.perform_in(interval, *args)
|
@@ -194,14 +214,14 @@ module Sidekiq
|
|
194
214
|
conn.pipelined do |pipeline|
|
195
215
|
atomic_push(pipeline, payloads)
|
196
216
|
end
|
197
|
-
rescue
|
217
|
+
rescue RedisClient::Error => ex
|
198
218
|
# 2550 Failover can cause the server to become a replica, need
|
199
219
|
# to disconnect and reopen the socket to get back to the primary.
|
200
220
|
# 4495 Use the same logic if we have a "Not enough replicas" error from the primary
|
201
221
|
# 4985 Use the same logic when a blocking command is force-unblocked
|
202
222
|
# The retry logic is copied from sidekiq.rb
|
203
223
|
if retryable && ex.message =~ /READONLY|NOREPLICAS|UNBLOCKED/
|
204
|
-
conn.
|
224
|
+
conn.close
|
205
225
|
retryable = false
|
206
226
|
retry
|
207
227
|
end
|
@@ -213,7 +233,7 @@ module Sidekiq
|
|
213
233
|
|
214
234
|
def atomic_push(conn, payloads)
|
215
235
|
if payloads.first.key?("at")
|
216
|
-
conn.zadd("schedule", payloads.
|
236
|
+
conn.zadd("schedule", payloads.flat_map { |hash|
|
217
237
|
at = hash.delete("at").to_s
|
218
238
|
[at, Sidekiq.dump_json(hash)]
|
219
239
|
})
|
@@ -224,17 +244,9 @@ module Sidekiq
|
|
224
244
|
entry["enqueued_at"] = now
|
225
245
|
Sidekiq.dump_json(entry)
|
226
246
|
}
|
227
|
-
conn.sadd("queues", queue)
|
247
|
+
conn.sadd("queues", [queue])
|
228
248
|
conn.lpush("queue:#{queue}", to_push)
|
229
249
|
end
|
230
250
|
end
|
231
|
-
|
232
|
-
def process_single(worker_class, item)
|
233
|
-
queue = item["queue"]
|
234
|
-
|
235
|
-
middleware.invoke(worker_class, item, queue, @redis_pool) do
|
236
|
-
item
|
237
|
-
end
|
238
|
-
end
|
239
251
|
end
|
240
252
|
end
|