sidekiq 4.2.4 → 6.2.1
Sign up to get free protection for your applications and to get access to all the features.
Potentially problematic release.
This version of sidekiq might be problematic. Click here for more details.
- checksums.yaml +5 -5
- data/Changes.md +445 -0
- data/LICENSE +1 -1
- data/README.md +21 -34
- data/bin/sidekiq +26 -2
- data/bin/sidekiqload +28 -38
- data/bin/sidekiqmon +8 -0
- data/lib/generators/sidekiq/templates/worker_spec.rb.erb +1 -1
- data/lib/generators/sidekiq/templates/worker_test.rb.erb +2 -2
- data/lib/generators/sidekiq/worker_generator.rb +21 -13
- data/lib/sidekiq/api.rb +347 -213
- data/lib/sidekiq/cli.rb +221 -212
- data/lib/sidekiq/client.rb +75 -52
- data/lib/sidekiq/delay.rb +41 -0
- data/lib/sidekiq/exception_handler.rb +12 -16
- data/lib/sidekiq/extensions/action_mailer.rb +13 -22
- data/lib/sidekiq/extensions/active_record.rb +13 -10
- data/lib/sidekiq/extensions/class_methods.rb +14 -11
- data/lib/sidekiq/extensions/generic_proxy.rb +10 -4
- data/lib/sidekiq/fetch.rb +38 -31
- data/lib/sidekiq/job_logger.rb +63 -0
- data/lib/sidekiq/job_retry.rb +263 -0
- data/lib/sidekiq/launcher.rb +169 -70
- data/lib/sidekiq/logger.rb +166 -0
- data/lib/sidekiq/manager.rb +17 -20
- data/lib/sidekiq/middleware/chain.rb +15 -5
- data/lib/sidekiq/middleware/i18n.rb +5 -7
- data/lib/sidekiq/monitor.rb +133 -0
- data/lib/sidekiq/paginator.rb +18 -14
- data/lib/sidekiq/processor.rb +161 -70
- data/lib/sidekiq/rails.rb +30 -73
- data/lib/sidekiq/redis_connection.rb +67 -20
- data/lib/sidekiq/scheduled.rb +61 -35
- data/lib/sidekiq/sd_notify.rb +149 -0
- data/lib/sidekiq/systemd.rb +24 -0
- data/lib/sidekiq/testing/inline.rb +2 -1
- data/lib/sidekiq/testing.rb +54 -26
- data/lib/sidekiq/util.rb +48 -15
- data/lib/sidekiq/version.rb +2 -1
- data/lib/sidekiq/web/action.rb +15 -15
- data/lib/sidekiq/web/application.rb +112 -89
- data/lib/sidekiq/web/csrf_protection.rb +180 -0
- data/lib/sidekiq/web/helpers.rb +153 -73
- data/lib/sidekiq/web/router.rb +27 -19
- data/lib/sidekiq/web.rb +64 -109
- data/lib/sidekiq/worker.rb +164 -41
- data/lib/sidekiq.rb +86 -60
- data/sidekiq.gemspec +24 -22
- data/web/assets/images/apple-touch-icon.png +0 -0
- data/web/assets/javascripts/application.js +25 -27
- data/web/assets/javascripts/dashboard.js +34 -38
- data/web/assets/stylesheets/application-dark.css +160 -0
- data/web/assets/stylesheets/application-rtl.css +246 -0
- data/web/assets/stylesheets/application.css +402 -12
- data/web/assets/stylesheets/bootstrap-rtl.min.css +9 -0
- data/web/assets/stylesheets/bootstrap.css +2 -2
- data/web/locales/ar.yml +81 -0
- data/web/locales/de.yml +14 -2
- data/web/locales/en.yml +4 -0
- data/web/locales/es.yml +4 -3
- data/web/locales/fa.yml +80 -0
- data/web/locales/fr.yml +3 -3
- data/web/locales/he.yml +79 -0
- data/web/locales/ja.yml +9 -4
- data/web/locales/lt.yml +83 -0
- data/web/locales/pl.yml +4 -4
- data/web/locales/ru.yml +4 -0
- data/web/locales/ur.yml +80 -0
- data/web/locales/vi.yml +83 -0
- data/web/views/_footer.erb +5 -2
- data/web/views/_job_info.erb +3 -2
- data/web/views/_nav.erb +4 -18
- data/web/views/_paging.erb +1 -1
- data/web/views/busy.erb +57 -19
- data/web/views/dashboard.erb +3 -3
- data/web/views/dead.erb +2 -2
- data/web/views/layout.erb +13 -2
- data/web/views/morgue.erb +19 -12
- data/web/views/queue.erb +22 -12
- data/web/views/queues.erb +13 -3
- data/web/views/retries.erb +22 -13
- data/web/views/retry.erb +3 -3
- data/web/views/scheduled.erb +7 -4
- metadata +42 -194
- data/.github/contributing.md +0 -32
- data/.github/issue_template.md +0 -4
- data/.gitignore +0 -12
- data/.travis.yml +0 -12
- data/3.0-Upgrade.md +0 -70
- data/4.0-Upgrade.md +0 -53
- data/COMM-LICENSE +0 -95
- data/Ent-Changes.md +0 -146
- data/Gemfile +0 -29
- data/Pro-2.0-Upgrade.md +0 -138
- data/Pro-3.0-Upgrade.md +0 -44
- data/Pro-Changes.md +0 -585
- data/Rakefile +0 -9
- data/bin/sidekiqctl +0 -99
- data/code_of_conduct.md +0 -50
- data/lib/sidekiq/core_ext.rb +0 -106
- data/lib/sidekiq/logging.rb +0 -106
- data/lib/sidekiq/middleware/server/active_record.rb +0 -13
- data/lib/sidekiq/middleware/server/logging.rb +0 -40
- data/lib/sidekiq/middleware/server/retry_jobs.rb +0 -205
- data/test/config.yml +0 -9
- data/test/env_based_config.yml +0 -11
- data/test/fake_env.rb +0 -1
- data/test/fixtures/en.yml +0 -2
- data/test/helper.rb +0 -75
- data/test/test_actors.rb +0 -138
- data/test/test_api.rb +0 -528
- data/test/test_cli.rb +0 -418
- data/test/test_client.rb +0 -266
- data/test/test_exception_handler.rb +0 -56
- data/test/test_extensions.rb +0 -127
- data/test/test_fetch.rb +0 -50
- data/test/test_launcher.rb +0 -95
- data/test/test_logging.rb +0 -35
- data/test/test_manager.rb +0 -50
- data/test/test_middleware.rb +0 -158
- data/test/test_processor.rb +0 -235
- data/test/test_rails.rb +0 -22
- data/test/test_redis_connection.rb +0 -132
- data/test/test_retry.rb +0 -326
- data/test/test_retry_exhausted.rb +0 -149
- data/test/test_scheduled.rb +0 -115
- data/test/test_scheduling.rb +0 -58
- data/test/test_sidekiq.rb +0 -107
- data/test/test_testing.rb +0 -143
- data/test/test_testing_fake.rb +0 -357
- data/test/test_testing_inline.rb +0 -94
- data/test/test_util.rb +0 -13
- data/test/test_web.rb +0 -726
- data/test/test_web_helpers.rb +0 -54
data/lib/sidekiq/processor.rb
CHANGED
@@ -1,9 +1,9 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
|
-
|
3
|
-
require
|
4
|
-
require
|
5
|
-
require
|
6
|
-
require
|
2
|
+
|
3
|
+
require "sidekiq/util"
|
4
|
+
require "sidekiq/fetch"
|
5
|
+
require "sidekiq/job_logger"
|
6
|
+
require "sidekiq/job_retry"
|
7
7
|
|
8
8
|
module Sidekiq
|
9
9
|
##
|
@@ -23,31 +23,32 @@ module Sidekiq
|
|
23
23
|
# to replace itself and exits.
|
24
24
|
#
|
25
25
|
class Processor
|
26
|
-
|
27
26
|
include Util
|
28
27
|
|
29
28
|
attr_reader :thread
|
30
29
|
attr_reader :job
|
31
30
|
|
32
|
-
def initialize(mgr)
|
31
|
+
def initialize(mgr, options)
|
33
32
|
@mgr = mgr
|
34
33
|
@down = false
|
35
34
|
@done = false
|
36
35
|
@job = nil
|
37
36
|
@thread = nil
|
38
|
-
@strategy =
|
39
|
-
@reloader =
|
37
|
+
@strategy = options[:fetch]
|
38
|
+
@reloader = options[:reloader] || proc { |&block| block.call }
|
39
|
+
@job_logger = (options[:job_logger] || Sidekiq::JobLogger).new
|
40
|
+
@retrier = Sidekiq::JobRetry.new
|
40
41
|
end
|
41
42
|
|
42
|
-
def terminate(wait=false)
|
43
|
+
def terminate(wait = false)
|
43
44
|
@done = true
|
44
|
-
return
|
45
|
+
return unless @thread
|
45
46
|
@thread.value if wait
|
46
47
|
end
|
47
48
|
|
48
|
-
def kill(wait=false)
|
49
|
+
def kill(wait = false)
|
49
50
|
@done = true
|
50
|
-
return
|
51
|
+
return unless @thread
|
51
52
|
# unlike the other actors, terminate does not wait
|
52
53
|
# for the thread to finish because we don't know how
|
53
54
|
# long the job will take to finish. Instead we
|
@@ -64,16 +65,12 @@ module Sidekiq
|
|
64
65
|
private unless $TESTING
|
65
66
|
|
66
67
|
def run
|
67
|
-
|
68
|
-
|
69
|
-
|
70
|
-
|
71
|
-
|
72
|
-
|
73
|
-
@mgr.processor_stopped(self)
|
74
|
-
rescue Exception => ex
|
75
|
-
@mgr.processor_died(self, ex)
|
76
|
-
end
|
68
|
+
process_one until @done
|
69
|
+
@mgr.processor_stopped(self)
|
70
|
+
rescue Sidekiq::Shutdown
|
71
|
+
@mgr.processor_stopped(self)
|
72
|
+
rescue Exception => ex
|
73
|
+
@mgr.processor_died(self, ex)
|
77
74
|
end
|
78
75
|
|
79
76
|
def process_one
|
@@ -83,14 +80,15 @@ module Sidekiq
|
|
83
80
|
end
|
84
81
|
|
85
82
|
def get_one
|
86
|
-
|
87
|
-
|
88
|
-
|
89
|
-
|
90
|
-
rescue Sidekiq::Shutdown
|
91
|
-
rescue => ex
|
92
|
-
handle_fetch_exception(ex)
|
83
|
+
work = @strategy.retrieve_work
|
84
|
+
if @down
|
85
|
+
logger.info { "Redis is online, #{::Process.clock_gettime(::Process::CLOCK_MONOTONIC) - @down} sec downtime" }
|
86
|
+
@down = nil
|
93
87
|
end
|
88
|
+
work
|
89
|
+
rescue Sidekiq::Shutdown
|
90
|
+
rescue => ex
|
91
|
+
handle_fetch_exception(ex)
|
94
92
|
end
|
95
93
|
|
96
94
|
def fetch
|
@@ -104,50 +102,93 @@ module Sidekiq
|
|
104
102
|
end
|
105
103
|
|
106
104
|
def handle_fetch_exception(ex)
|
107
|
-
|
108
|
-
@down =
|
105
|
+
unless @down
|
106
|
+
@down = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
|
109
107
|
logger.error("Error fetching job: #{ex}")
|
110
|
-
ex
|
111
|
-
logger.error(bt)
|
112
|
-
end
|
108
|
+
handle_exception(ex)
|
113
109
|
end
|
114
110
|
sleep(1)
|
115
111
|
nil
|
116
112
|
end
|
117
113
|
|
114
|
+
def dispatch(job_hash, queue, jobstr)
|
115
|
+
# since middleware can mutate the job hash
|
116
|
+
# we need to clone it to report the original
|
117
|
+
# job structure to the Web UI
|
118
|
+
# or to push back to redis when retrying.
|
119
|
+
# To avoid costly and, most of the time, useless cloning here,
|
120
|
+
# we pass original String of JSON to respected methods
|
121
|
+
# to re-parse it there if we need access to the original, untouched job
|
122
|
+
|
123
|
+
@job_logger.prepare(job_hash) do
|
124
|
+
@retrier.global(jobstr, queue) do
|
125
|
+
@job_logger.call(job_hash, queue) do
|
126
|
+
stats(jobstr, queue) do
|
127
|
+
# Rails 5 requires a Reloader to wrap code execution. In order to
|
128
|
+
# constantize the worker and instantiate an instance, we have to call
|
129
|
+
# the Reloader. It handles code loading, db connection management, etc.
|
130
|
+
# Effectively this block denotes a "unit of work" to Rails.
|
131
|
+
@reloader.call do
|
132
|
+
klass = constantize(job_hash["class"])
|
133
|
+
worker = klass.new
|
134
|
+
worker.jid = job_hash["jid"]
|
135
|
+
@retrier.local(worker, jobstr, queue) do
|
136
|
+
yield worker
|
137
|
+
end
|
138
|
+
end
|
139
|
+
end
|
140
|
+
end
|
141
|
+
end
|
142
|
+
end
|
143
|
+
end
|
144
|
+
|
118
145
|
def process(work)
|
119
146
|
jobstr = work.job
|
120
147
|
queue = work.queue_name
|
121
148
|
|
149
|
+
# Treat malformed JSON as a special case: job goes straight to the morgue.
|
150
|
+
job_hash = nil
|
151
|
+
begin
|
152
|
+
job_hash = Sidekiq.load_json(jobstr)
|
153
|
+
rescue => ex
|
154
|
+
handle_exception(ex, {context: "Invalid JSON for job", jobstr: jobstr})
|
155
|
+
# we can't notify because the job isn't a valid hash payload.
|
156
|
+
DeadSet.new.kill(jobstr, notify_failure: false)
|
157
|
+
return work.acknowledge
|
158
|
+
end
|
159
|
+
|
122
160
|
ack = false
|
123
161
|
begin
|
124
|
-
|
125
|
-
|
126
|
-
|
127
|
-
worker = klass.new
|
128
|
-
worker.jid = job['jid'.freeze]
|
129
|
-
|
130
|
-
stats(worker, job, queue) do
|
131
|
-
Sidekiq.server_middleware.invoke(worker, job, queue) do
|
132
|
-
# Only ack if we either attempted to start this job or
|
133
|
-
# successfully completed it. This prevents us from
|
134
|
-
# losing jobs if a middleware raises an exception before yielding
|
135
|
-
ack = true
|
136
|
-
execute_job(worker, cloned(job['args'.freeze]))
|
137
|
-
end
|
162
|
+
dispatch(job_hash, queue, jobstr) do |worker|
|
163
|
+
Sidekiq.server_middleware.invoke(worker, job_hash, queue) do
|
164
|
+
execute_job(worker, job_hash["args"])
|
138
165
|
end
|
139
|
-
ack = true
|
140
166
|
end
|
167
|
+
ack = true
|
141
168
|
rescue Sidekiq::Shutdown
|
142
169
|
# Had to force kill this job because it didn't finish
|
143
170
|
# within the timeout. Don't acknowledge the work since
|
144
171
|
# we didn't properly finish it.
|
145
|
-
|
172
|
+
rescue Sidekiq::JobRetry::Handled => h
|
173
|
+
# this is the common case: job raised error and Sidekiq::JobRetry::Handled
|
174
|
+
# signals that we created a retry successfully. We can acknowlege the job.
|
175
|
+
ack = true
|
176
|
+
e = h.cause || h
|
177
|
+
handle_exception(e, {context: "Job raised exception", job: job_hash, jobstr: jobstr})
|
178
|
+
raise e
|
146
179
|
rescue Exception => ex
|
147
|
-
|
148
|
-
|
180
|
+
# Unexpected error! This is very bad and indicates an exception that got past
|
181
|
+
# the retry subsystem (e.g. network partition). We won't acknowledge the job
|
182
|
+
# so it can be rescued when using Sidekiq Pro.
|
183
|
+
handle_exception(ex, {context: "Internal exception!", job: job_hash, jobstr: jobstr})
|
184
|
+
raise ex
|
149
185
|
ensure
|
150
|
-
|
186
|
+
if ack
|
187
|
+
# We don't want a shutdown signal to interrupt job acknowledgment.
|
188
|
+
Thread.handle_interrupt(Sidekiq::Shutdown => :never) do
|
189
|
+
work.acknowledge
|
190
|
+
end
|
191
|
+
end
|
151
192
|
end
|
152
193
|
end
|
153
194
|
|
@@ -155,35 +196,85 @@ module Sidekiq
|
|
155
196
|
worker.perform(*cloned_args)
|
156
197
|
end
|
157
198
|
|
158
|
-
|
159
|
-
|
199
|
+
# Ruby doesn't provide atomic counters out of the box so we'll
|
200
|
+
# implement something simple ourselves.
|
201
|
+
# https://bugs.ruby-lang.org/issues/14706
|
202
|
+
class Counter
|
203
|
+
def initialize
|
204
|
+
@value = 0
|
205
|
+
@lock = Mutex.new
|
206
|
+
end
|
207
|
+
|
208
|
+
def incr(amount = 1)
|
209
|
+
@lock.synchronize { @value += amount }
|
210
|
+
end
|
211
|
+
|
212
|
+
def reset
|
213
|
+
@lock.synchronize {
|
214
|
+
val = @value
|
215
|
+
@value = 0
|
216
|
+
val
|
217
|
+
}
|
218
|
+
end
|
219
|
+
end
|
220
|
+
|
221
|
+
# jruby's Hash implementation is not threadsafe, so we wrap it in a mutex here
|
222
|
+
class SharedWorkerState
|
223
|
+
def initialize
|
224
|
+
@worker_state = {}
|
225
|
+
@lock = Mutex.new
|
226
|
+
end
|
227
|
+
|
228
|
+
def set(tid, hash)
|
229
|
+
@lock.synchronize { @worker_state[tid] = hash }
|
230
|
+
end
|
231
|
+
|
232
|
+
def delete(tid)
|
233
|
+
@lock.synchronize { @worker_state.delete(tid) }
|
234
|
+
end
|
235
|
+
|
236
|
+
def dup
|
237
|
+
@lock.synchronize { @worker_state.dup }
|
238
|
+
end
|
239
|
+
|
240
|
+
def size
|
241
|
+
@lock.synchronize { @worker_state.size }
|
242
|
+
end
|
243
|
+
|
244
|
+
def clear
|
245
|
+
@lock.synchronize { @worker_state.clear }
|
246
|
+
end
|
160
247
|
end
|
161
248
|
|
162
|
-
|
163
|
-
|
164
|
-
|
249
|
+
PROCESSED = Counter.new
|
250
|
+
FAILURE = Counter.new
|
251
|
+
WORKER_STATE = SharedWorkerState.new
|
165
252
|
|
166
|
-
def stats(
|
167
|
-
tid
|
168
|
-
WORKER_STATE[tid] = {:queue => queue, :payload => cloned(job), :run_at => Time.now.to_i }
|
253
|
+
def stats(jobstr, queue)
|
254
|
+
WORKER_STATE.set(tid, {queue: queue, payload: jobstr, run_at: Time.now.to_i})
|
169
255
|
|
170
256
|
begin
|
171
257
|
yield
|
172
258
|
rescue Exception
|
173
|
-
FAILURE.
|
259
|
+
FAILURE.incr
|
174
260
|
raise
|
175
261
|
ensure
|
176
262
|
WORKER_STATE.delete(tid)
|
177
|
-
PROCESSED.
|
263
|
+
PROCESSED.incr
|
178
264
|
end
|
179
265
|
end
|
180
266
|
|
181
|
-
|
182
|
-
|
183
|
-
# been mutated by the worker.
|
184
|
-
def cloned(ary)
|
185
|
-
Marshal.load(Marshal.dump(ary))
|
186
|
-
end
|
267
|
+
def constantize(str)
|
268
|
+
return Object.const_get(str) unless str.include?("::")
|
187
269
|
|
270
|
+
names = str.split("::")
|
271
|
+
names.shift if names.empty? || names.first.empty?
|
272
|
+
|
273
|
+
names.inject(Object) do |constant, name|
|
274
|
+
# the false flag limits search for name to under the constant namespace
|
275
|
+
# which mimics Rails' behaviour
|
276
|
+
constant.const_get(name, false)
|
277
|
+
end
|
278
|
+
end
|
188
279
|
end
|
189
280
|
end
|
data/lib/sidekiq/rails.rb
CHANGED
@@ -1,81 +1,11 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
|
-
module Sidekiq
|
3
|
-
def self.hook_rails!
|
4
|
-
return if defined?(@delay_removed)
|
5
|
-
|
6
|
-
ActiveSupport.on_load(:active_record) do
|
7
|
-
include Sidekiq::Extensions::ActiveRecord
|
8
|
-
end
|
9
|
-
|
10
|
-
ActiveSupport.on_load(:action_mailer) do
|
11
|
-
extend Sidekiq::Extensions::ActionMailer
|
12
|
-
end
|
13
2
|
|
14
|
-
|
15
|
-
end
|
16
|
-
|
17
|
-
# Removes the generic aliases which MAY clash with names of already
|
18
|
-
# created methods by other applications. The methods `sidekiq_delay`,
|
19
|
-
# `sidekiq_delay_for` and `sidekiq_delay_until` can be used instead.
|
20
|
-
def self.remove_delay!
|
21
|
-
@delay_removed = true
|
22
|
-
|
23
|
-
[Extensions::ActiveRecord,
|
24
|
-
Extensions::ActionMailer,
|
25
|
-
Extensions::Klass].each do |mod|
|
26
|
-
mod.module_eval do
|
27
|
-
remove_method :delay if respond_to?(:delay)
|
28
|
-
remove_method :delay_for if respond_to?(:delay_for)
|
29
|
-
remove_method :delay_until if respond_to?(:delay_until)
|
30
|
-
end
|
31
|
-
end
|
32
|
-
end
|
3
|
+
require "sidekiq/worker"
|
33
4
|
|
5
|
+
module Sidekiq
|
34
6
|
class Rails < ::Rails::Engine
|
35
|
-
# We need to setup this up before any application configuration which might
|
36
|
-
# change Sidekiq middleware.
|
37
|
-
#
|
38
|
-
# This hook happens after `Rails::Application` is inherited within
|
39
|
-
# config/application.rb and before config is touched, usually within the
|
40
|
-
# class block. Definitely before config/environments/*.rb and
|
41
|
-
# config/initializers/*.rb.
|
42
|
-
config.before_configuration do
|
43
|
-
if ::Rails::VERSION::MAJOR < 5 && defined?(::ActiveRecord)
|
44
|
-
Sidekiq.server_middleware do |chain|
|
45
|
-
require 'sidekiq/middleware/server/active_record'
|
46
|
-
chain.add Sidekiq::Middleware::Server::ActiveRecord
|
47
|
-
end
|
48
|
-
end
|
49
|
-
end
|
50
|
-
|
51
|
-
initializer 'sidekiq' do
|
52
|
-
Sidekiq.hook_rails!
|
53
|
-
end
|
54
|
-
|
55
|
-
# We have to add the reloader after initialize to see if cache_classes has
|
56
|
-
# been turned on.
|
57
|
-
#
|
58
|
-
# This hook happens after all initialziers are run, just before returning
|
59
|
-
# from config/environment.rb back to sidekiq/cli.rb.
|
60
|
-
config.after_initialize do
|
61
|
-
if ::Rails::VERSION::MAJOR >= 5
|
62
|
-
# The reloader also takes care of ActiveRecord but is incompatible with
|
63
|
-
# the ActiveRecord middleware so make sure it's not in the chain already.
|
64
|
-
if defined?(Sidekiq::Middleware::Server::ActiveRecord) && Sidekiq.server_middleware.exists?(Sidekiq::Middleware::Server::ActiveRecord)
|
65
|
-
raise ArgumentError, "You are using the Sidekiq ActiveRecord middleware and the new Rails 5 reloader which are incompatible. Please remove the ActiveRecord middleware from your Sidekiq middleware configuration."
|
66
|
-
elsif ::Rails.application.config.cache_classes
|
67
|
-
# The reloader API has proven to be troublesome under load in production.
|
68
|
-
# We won't use it at all when classes are cached, see #3154
|
69
|
-
Sidekiq.logger.debug { "Autoload disabled in #{::Rails.env}, Sidekiq will not reload changed classes" }
|
70
|
-
else
|
71
|
-
Sidekiq.options[:reloader] = Sidekiq::Rails::Reloader.new
|
72
|
-
end
|
73
|
-
end
|
74
|
-
end
|
75
|
-
|
76
7
|
class Reloader
|
77
8
|
def initialize(app = ::Rails.application)
|
78
|
-
Sidekiq.logger.debug "Enabling Rails 5+ live code reloading, so hot!" unless app.config.cache_classes
|
79
9
|
@app = app
|
80
10
|
end
|
81
11
|
|
@@ -89,5 +19,32 @@ module Sidekiq
|
|
89
19
|
"#<Sidekiq::Rails::Reloader @app=#{@app.class.name}>"
|
90
20
|
end
|
91
21
|
end
|
92
|
-
|
22
|
+
|
23
|
+
# By including the Options module, we allow AJs to directly control sidekiq features
|
24
|
+
# via the *sidekiq_options* class method and, for instance, not use AJ's retry system.
|
25
|
+
# AJ retries don't show up in the Sidekiq UI Retries tab, save any error data, can't be
|
26
|
+
# manually retried, don't automatically die, etc.
|
27
|
+
#
|
28
|
+
# class SomeJob < ActiveJob::Base
|
29
|
+
# queue_as :default
|
30
|
+
# sidekiq_options retry: 3, backtrace: 10
|
31
|
+
# def perform
|
32
|
+
# end
|
33
|
+
# end
|
34
|
+
initializer "sidekiq.active_job_integration" do
|
35
|
+
ActiveSupport.on_load(:active_job) do
|
36
|
+
include ::Sidekiq::Worker::Options unless respond_to?(:sidekiq_options)
|
37
|
+
end
|
38
|
+
end
|
39
|
+
|
40
|
+
# This hook happens after all initializers are run, just before returning
|
41
|
+
# from config/environment.rb back to sidekiq/cli.rb.
|
42
|
+
#
|
43
|
+
# None of this matters on the client-side, only within the Sidekiq process itself.
|
44
|
+
config.after_initialize do
|
45
|
+
Sidekiq.configure_server do |_|
|
46
|
+
Sidekiq.options[:reloader] = Sidekiq::Rails::Reloader.new
|
47
|
+
end
|
48
|
+
end
|
49
|
+
end
|
93
50
|
end
|
@@ -1,26 +1,38 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
|
-
|
3
|
-
require
|
4
|
-
require
|
2
|
+
|
3
|
+
require "connection_pool"
|
4
|
+
require "redis"
|
5
|
+
require "uri"
|
5
6
|
|
6
7
|
module Sidekiq
|
7
8
|
class RedisConnection
|
8
9
|
class << self
|
10
|
+
def create(options = {})
|
11
|
+
symbolized_options = options.transform_keys(&:to_sym)
|
9
12
|
|
10
|
-
|
11
|
-
|
12
|
-
|
13
|
-
options[:url] ||= determine_redis_provider
|
13
|
+
if !symbolized_options[:url] && (u = determine_redis_provider)
|
14
|
+
symbolized_options[:url] = u
|
15
|
+
end
|
14
16
|
|
15
|
-
size =
|
17
|
+
size = if symbolized_options[:size]
|
18
|
+
symbolized_options[:size]
|
19
|
+
elsif Sidekiq.server?
|
20
|
+
# Give ourselves plenty of connections. pool is lazy
|
21
|
+
# so we won't create them until we need them.
|
22
|
+
Sidekiq.options[:concurrency] + 5
|
23
|
+
elsif ENV["RAILS_MAX_THREADS"]
|
24
|
+
Integer(ENV["RAILS_MAX_THREADS"])
|
25
|
+
else
|
26
|
+
5
|
27
|
+
end
|
16
28
|
|
17
29
|
verify_sizing(size, Sidekiq.options[:concurrency]) if Sidekiq.server?
|
18
30
|
|
19
|
-
pool_timeout =
|
20
|
-
log_info(
|
31
|
+
pool_timeout = symbolized_options[:pool_timeout] || 1
|
32
|
+
log_info(symbolized_options)
|
21
33
|
|
22
|
-
ConnectionPool.new(:
|
23
|
-
build_client(
|
34
|
+
ConnectionPool.new(timeout: pool_timeout, size: size) do
|
35
|
+
build_client(symbolized_options)
|
24
36
|
end
|
25
37
|
end
|
26
38
|
|
@@ -35,7 +47,7 @@ module Sidekiq
|
|
35
47
|
# - enterprise's leader election
|
36
48
|
# - enterprise's cron support
|
37
49
|
def verify_sizing(size, concurrency)
|
38
|
-
raise ArgumentError, "Your Redis connection pool is too small for Sidekiq to work. Your pool has #{size} connections but
|
50
|
+
raise ArgumentError, "Your Redis connection pool is too small for Sidekiq to work. Your pool has #{size} connections but must have at least #{concurrency + 2}" if size < (concurrency + 2)
|
39
51
|
end
|
40
52
|
|
41
53
|
def build_client(options)
|
@@ -44,8 +56,8 @@ module Sidekiq
|
|
44
56
|
client = Redis.new client_opts(options)
|
45
57
|
if namespace
|
46
58
|
begin
|
47
|
-
require
|
48
|
-
Redis::Namespace.new(namespace, :
|
59
|
+
require "redis/namespace"
|
60
|
+
Redis::Namespace.new(namespace, redis: client)
|
49
61
|
rescue LoadError
|
50
62
|
Sidekiq.logger.error("Your Redis configuration uses the namespace '#{namespace}' but the redis-namespace gem is not included in the Gemfile." \
|
51
63
|
"Add the gem to your Gemfile to continue using a namespace. Otherwise, remove the namespace parameter.")
|
@@ -67,15 +79,28 @@ module Sidekiq
|
|
67
79
|
opts.delete(:network_timeout)
|
68
80
|
end
|
69
81
|
|
70
|
-
opts[:driver]
|
82
|
+
opts[:driver] ||= Redis::Connection.drivers.last || "ruby"
|
83
|
+
|
84
|
+
# Issue #3303, redis-rb will silently retry an operation.
|
85
|
+
# This can lead to duplicate jobs if Sidekiq::Client's LPUSH
|
86
|
+
# is performed twice but I believe this is much, much rarer
|
87
|
+
# than the reconnect silently fixing a problem; we keep it
|
88
|
+
# on by default.
|
89
|
+
opts[:reconnect_attempts] ||= 1
|
71
90
|
|
72
91
|
opts
|
73
92
|
end
|
74
93
|
|
75
94
|
def log_info(options)
|
76
|
-
# Don't log Redis AUTH password
|
77
95
|
redacted = "REDACTED"
|
78
|
-
|
96
|
+
|
97
|
+
# deep clone so we can muck with these options all we want
|
98
|
+
#
|
99
|
+
# exclude SSL params from dump-and-load because some information isn't
|
100
|
+
# safely dumpable in current Rubies
|
101
|
+
keys = options.keys
|
102
|
+
keys.delete(:ssl_params)
|
103
|
+
scrubbed_options = Marshal.load(Marshal.dump(options.slice(*keys)))
|
79
104
|
if scrubbed_options[:url] && (uri = URI.parse(scrubbed_options[:url])) && uri.password
|
80
105
|
uri.password = redacted
|
81
106
|
scrubbed_options[:url] = uri.to_s
|
@@ -83,6 +108,9 @@ module Sidekiq
|
|
83
108
|
if scrubbed_options[:password]
|
84
109
|
scrubbed_options[:password] = redacted
|
85
110
|
end
|
111
|
+
scrubbed_options[:sentinels]&.each do |sentinel|
|
112
|
+
sentinel[:password] = redacted if sentinel[:password]
|
113
|
+
end
|
86
114
|
if Sidekiq.server?
|
87
115
|
Sidekiq.logger.info("Booting Sidekiq #{Sidekiq::VERSION} with redis options #{scrubbed_options}")
|
88
116
|
else
|
@@ -91,9 +119,28 @@ module Sidekiq
|
|
91
119
|
end
|
92
120
|
|
93
121
|
def determine_redis_provider
|
94
|
-
|
95
|
-
|
122
|
+
# If you have this in your environment:
|
123
|
+
# MY_REDIS_URL=redis://hostname.example.com:1238/4
|
124
|
+
# then set:
|
125
|
+
# REDIS_PROVIDER=MY_REDIS_URL
|
126
|
+
# and Sidekiq will find your custom URL variable with no custom
|
127
|
+
# initialization code at all.
|
128
|
+
#
|
129
|
+
p = ENV["REDIS_PROVIDER"]
|
130
|
+
if p && p =~ /:/
|
131
|
+
raise <<~EOM
|
132
|
+
REDIS_PROVIDER should be set to the name of the variable which contains the Redis URL, not a URL itself.
|
133
|
+
Platforms like Heroku will sell addons that publish a *_URL variable. You need to tell Sidekiq with REDIS_PROVIDER, e.g.:
|
134
|
+
|
135
|
+
REDISTOGO_URL=redis://somehost.example.com:6379/4
|
136
|
+
REDIS_PROVIDER=REDISTOGO_URL
|
137
|
+
EOM
|
138
|
+
end
|
96
139
|
|
140
|
+
ENV[
|
141
|
+
p || "REDIS_URL"
|
142
|
+
]
|
143
|
+
end
|
97
144
|
end
|
98
145
|
end
|
99
146
|
end
|