sidekiq 6.0.1 → 6.2.2
Sign up to get free protection for your applications and to get access to all the features.
Potentially problematic release.
This version of sidekiq might be problematic. Click here for more details.
- checksums.yaml +4 -4
- data/Changes.md +147 -2
- data/LICENSE +1 -1
- data/README.md +4 -7
- data/bin/sidekiq +26 -2
- data/lib/generators/sidekiq/worker_generator.rb +1 -1
- data/lib/sidekiq/api.rb +151 -111
- data/lib/sidekiq/cli.rb +39 -10
- data/lib/sidekiq/client.rb +26 -15
- data/lib/sidekiq/extensions/action_mailer.rb +3 -2
- data/lib/sidekiq/extensions/active_record.rb +4 -3
- data/lib/sidekiq/extensions/class_methods.rb +5 -4
- data/lib/sidekiq/extensions/generic_proxy.rb +3 -1
- data/lib/sidekiq/fetch.rb +29 -21
- data/lib/sidekiq/job.rb +8 -0
- data/lib/sidekiq/job_logger.rb +2 -2
- data/lib/sidekiq/job_retry.rb +11 -12
- data/lib/sidekiq/launcher.rb +104 -24
- data/lib/sidekiq/logger.rb +12 -11
- data/lib/sidekiq/manager.rb +4 -4
- data/lib/sidekiq/middleware/chain.rb +6 -4
- data/lib/sidekiq/monitor.rb +2 -17
- data/lib/sidekiq/processor.rb +17 -39
- data/lib/sidekiq/rails.rb +16 -18
- data/lib/sidekiq/redis_connection.rb +21 -13
- data/lib/sidekiq/scheduled.rb +7 -1
- data/lib/sidekiq/sd_notify.rb +149 -0
- data/lib/sidekiq/systemd.rb +24 -0
- data/lib/sidekiq/testing.rb +2 -4
- data/lib/sidekiq/util.rb +28 -2
- data/lib/sidekiq/version.rb +1 -1
- data/lib/sidekiq/web/action.rb +2 -2
- data/lib/sidekiq/web/application.rb +30 -19
- data/lib/sidekiq/web/csrf_protection.rb +180 -0
- data/lib/sidekiq/web/helpers.rb +35 -24
- data/lib/sidekiq/web/router.rb +6 -5
- data/lib/sidekiq/web.rb +37 -73
- data/lib/sidekiq/worker.rb +4 -7
- data/lib/sidekiq.rb +14 -8
- data/sidekiq.gemspec +12 -5
- data/web/assets/images/apple-touch-icon.png +0 -0
- data/web/assets/javascripts/application.js +25 -27
- data/web/assets/stylesheets/application-dark.css +146 -124
- data/web/assets/stylesheets/application.css +35 -135
- data/web/locales/ar.yml +8 -2
- data/web/locales/de.yml +14 -2
- data/web/locales/en.yml +5 -0
- data/web/locales/es.yml +18 -2
- data/web/locales/fr.yml +10 -3
- data/web/locales/ja.yml +5 -0
- data/web/locales/lt.yml +83 -0
- data/web/locales/pl.yml +4 -4
- data/web/locales/ru.yml +4 -0
- data/web/locales/vi.yml +83 -0
- data/web/views/_job_info.erb +1 -1
- data/web/views/busy.erb +50 -19
- data/web/views/dashboard.erb +14 -6
- data/web/views/dead.erb +1 -1
- data/web/views/layout.erb +2 -1
- data/web/views/morgue.erb +6 -6
- data/web/views/queue.erb +1 -1
- data/web/views/queues.erb +10 -2
- data/web/views/retries.erb +7 -7
- data/web/views/retry.erb +1 -1
- data/web/views/scheduled.erb +1 -1
- metadata +26 -50
- data/.circleci/config.yml +0 -82
- data/.github/contributing.md +0 -32
- data/.github/issue_template.md +0 -11
- data/.gitignore +0 -13
- data/.standard.yml +0 -20
- data/3.0-Upgrade.md +0 -70
- data/4.0-Upgrade.md +0 -53
- data/5.0-Upgrade.md +0 -56
- data/6.0-Upgrade.md +0 -72
- data/COMM-LICENSE +0 -97
- data/Ent-2.0-Upgrade.md +0 -37
- data/Ent-Changes.md +0 -256
- data/Gemfile +0 -24
- data/Gemfile.lock +0 -196
- data/Pro-2.0-Upgrade.md +0 -138
- data/Pro-3.0-Upgrade.md +0 -44
- data/Pro-4.0-Upgrade.md +0 -35
- data/Pro-5.0-Upgrade.md +0 -25
- data/Pro-Changes.md +0 -776
- data/Rakefile +0 -10
- data/code_of_conduct.md +0 -50
data/lib/sidekiq/processor.rb
CHANGED
@@ -28,15 +28,15 @@ module Sidekiq
|
|
28
28
|
attr_reader :thread
|
29
29
|
attr_reader :job
|
30
30
|
|
31
|
-
def initialize(mgr)
|
31
|
+
def initialize(mgr, options)
|
32
32
|
@mgr = mgr
|
33
33
|
@down = false
|
34
34
|
@done = false
|
35
35
|
@job = nil
|
36
36
|
@thread = nil
|
37
|
-
@strategy =
|
38
|
-
@reloader =
|
39
|
-
@job_logger = (
|
37
|
+
@strategy = options[:fetch]
|
38
|
+
@reloader = options[:reloader] || proc { |&block| block.call }
|
39
|
+
@job_logger = (options[:job_logger] || Sidekiq::JobLogger).new
|
40
40
|
@retrier = Sidekiq::JobRetry.new
|
41
41
|
end
|
42
42
|
|
@@ -111,16 +111,19 @@ module Sidekiq
|
|
111
111
|
nil
|
112
112
|
end
|
113
113
|
|
114
|
-
def dispatch(job_hash, queue)
|
114
|
+
def dispatch(job_hash, queue, jobstr)
|
115
115
|
# since middleware can mutate the job hash
|
116
|
-
# we clone
|
116
|
+
# we need to clone it to report the original
|
117
117
|
# job structure to the Web UI
|
118
|
-
|
118
|
+
# or to push back to redis when retrying.
|
119
|
+
# To avoid costly and, most of the time, useless cloning here,
|
120
|
+
# we pass original String of JSON to respected methods
|
121
|
+
# to re-parse it there if we need access to the original, untouched job
|
119
122
|
|
120
123
|
@job_logger.prepare(job_hash) do
|
121
|
-
@retrier.global(
|
124
|
+
@retrier.global(jobstr, queue) do
|
122
125
|
@job_logger.call(job_hash, queue) do
|
123
|
-
stats(
|
126
|
+
stats(jobstr, queue) do
|
124
127
|
# Rails 5 requires a Reloader to wrap code execution. In order to
|
125
128
|
# constantize the worker and instantiate an instance, we have to call
|
126
129
|
# the Reloader. It handles code loading, db connection management, etc.
|
@@ -129,7 +132,7 @@ module Sidekiq
|
|
129
132
|
klass = constantize(job_hash["class"])
|
130
133
|
worker = klass.new
|
131
134
|
worker.jid = job_hash["jid"]
|
132
|
-
@retrier.local(worker,
|
135
|
+
@retrier.local(worker, jobstr, queue) do
|
133
136
|
yield worker
|
134
137
|
end
|
135
138
|
end
|
@@ -156,7 +159,7 @@ module Sidekiq
|
|
156
159
|
|
157
160
|
ack = false
|
158
161
|
begin
|
159
|
-
dispatch(job_hash, queue) do |worker|
|
162
|
+
dispatch(job_hash, queue, jobstr) do |worker|
|
160
163
|
Sidekiq.server_middleware.invoke(worker, job_hash, queue) do
|
161
164
|
execute_job(worker, job_hash["args"])
|
162
165
|
end
|
@@ -178,7 +181,7 @@ module Sidekiq
|
|
178
181
|
# the retry subsystem (e.g. network partition). We won't acknowledge the job
|
179
182
|
# so it can be rescued when using Sidekiq Pro.
|
180
183
|
handle_exception(ex, {context: "Internal exception!", job: job_hash, jobstr: jobstr})
|
181
|
-
raise
|
184
|
+
raise ex
|
182
185
|
ensure
|
183
186
|
if ack
|
184
187
|
# We don't want a shutdown signal to interrupt job acknowledgment.
|
@@ -247,8 +250,8 @@ module Sidekiq
|
|
247
250
|
FAILURE = Counter.new
|
248
251
|
WORKER_STATE = SharedWorkerState.new
|
249
252
|
|
250
|
-
def stats(
|
251
|
-
WORKER_STATE.set(tid, {queue: queue, payload:
|
253
|
+
def stats(jobstr, queue)
|
254
|
+
WORKER_STATE.set(tid, {queue: queue, payload: jobstr, run_at: Time.now.to_i})
|
252
255
|
|
253
256
|
begin
|
254
257
|
yield
|
@@ -273,30 +276,5 @@ module Sidekiq
|
|
273
276
|
constant.const_get(name, false)
|
274
277
|
end
|
275
278
|
end
|
276
|
-
|
277
|
-
# Deep clone the arguments passed to the worker so that if
|
278
|
-
# the job fails, what is pushed back onto Redis hasn't
|
279
|
-
# been mutated by the worker.
|
280
|
-
def json_clone(obj)
|
281
|
-
if Integer === obj || Float === obj || TrueClass === obj || FalseClass === obj || NilClass === obj
|
282
|
-
return obj
|
283
|
-
elsif String === obj
|
284
|
-
return obj.dup
|
285
|
-
elsif Array === obj
|
286
|
-
duped = Array.new(obj.size)
|
287
|
-
obj.each_with_index do |value, index|
|
288
|
-
duped[index] = json_clone(value)
|
289
|
-
end
|
290
|
-
elsif Hash === obj
|
291
|
-
duped = obj.dup
|
292
|
-
duped.each_pair do |key, value|
|
293
|
-
duped[key] = json_clone(value)
|
294
|
-
end
|
295
|
-
else
|
296
|
-
duped = obj.dup
|
297
|
-
end
|
298
|
-
|
299
|
-
duped
|
300
|
-
end
|
301
279
|
end
|
302
280
|
end
|
data/lib/sidekiq/rails.rb
CHANGED
@@ -4,6 +4,22 @@ require "sidekiq/worker"
|
|
4
4
|
|
5
5
|
module Sidekiq
|
6
6
|
class Rails < ::Rails::Engine
|
7
|
+
class Reloader
|
8
|
+
def initialize(app = ::Rails.application)
|
9
|
+
@app = app
|
10
|
+
end
|
11
|
+
|
12
|
+
def call
|
13
|
+
@app.reloader.wrap do
|
14
|
+
yield
|
15
|
+
end
|
16
|
+
end
|
17
|
+
|
18
|
+
def inspect
|
19
|
+
"#<Sidekiq::Rails::Reloader @app=#{@app.class.name}>"
|
20
|
+
end
|
21
|
+
end
|
22
|
+
|
7
23
|
# By including the Options module, we allow AJs to directly control sidekiq features
|
8
24
|
# via the *sidekiq_options* class method and, for instance, not use AJ's retry system.
|
9
25
|
# AJ retries don't show up in the Sidekiq UI Retries tab, save any error data, can't be
|
@@ -23,8 +39,6 @@ module Sidekiq
|
|
23
39
|
|
24
40
|
# This hook happens after all initializers are run, just before returning
|
25
41
|
# from config/environment.rb back to sidekiq/cli.rb.
|
26
|
-
# We have to add the reloader after initialize to see if cache_classes has
|
27
|
-
# been turned on.
|
28
42
|
#
|
29
43
|
# None of this matters on the client-side, only within the Sidekiq process itself.
|
30
44
|
config.after_initialize do
|
@@ -32,21 +46,5 @@ module Sidekiq
|
|
32
46
|
Sidekiq.options[:reloader] = Sidekiq::Rails::Reloader.new
|
33
47
|
end
|
34
48
|
end
|
35
|
-
|
36
|
-
class Reloader
|
37
|
-
def initialize(app = ::Rails.application)
|
38
|
-
@app = app
|
39
|
-
end
|
40
|
-
|
41
|
-
def call
|
42
|
-
@app.reloader.wrap do
|
43
|
-
yield
|
44
|
-
end
|
45
|
-
end
|
46
|
-
|
47
|
-
def inspect
|
48
|
-
"#<Sidekiq::Rails::Reloader @app=#{@app.class.name}>"
|
49
|
-
end
|
50
|
-
end
|
51
49
|
end
|
52
50
|
end
|
@@ -8,15 +8,14 @@ module Sidekiq
|
|
8
8
|
class RedisConnection
|
9
9
|
class << self
|
10
10
|
def create(options = {})
|
11
|
-
options.
|
12
|
-
options[key.to_sym] = options.delete(key)
|
13
|
-
end
|
11
|
+
symbolized_options = options.transform_keys(&:to_sym)
|
14
12
|
|
15
|
-
|
16
|
-
|
13
|
+
if !symbolized_options[:url] && (u = determine_redis_provider)
|
14
|
+
symbolized_options[:url] = u
|
15
|
+
end
|
17
16
|
|
18
|
-
size = if
|
19
|
-
|
17
|
+
size = if symbolized_options[:size]
|
18
|
+
symbolized_options[:size]
|
20
19
|
elsif Sidekiq.server?
|
21
20
|
# Give ourselves plenty of connections. pool is lazy
|
22
21
|
# so we won't create them until we need them.
|
@@ -29,11 +28,11 @@ module Sidekiq
|
|
29
28
|
|
30
29
|
verify_sizing(size, Sidekiq.options[:concurrency]) if Sidekiq.server?
|
31
30
|
|
32
|
-
pool_timeout =
|
33
|
-
log_info(
|
31
|
+
pool_timeout = symbolized_options[:pool_timeout] || 1
|
32
|
+
log_info(symbolized_options)
|
34
33
|
|
35
34
|
ConnectionPool.new(timeout: pool_timeout, size: size) do
|
36
|
-
build_client(
|
35
|
+
build_client(symbolized_options)
|
37
36
|
end
|
38
37
|
end
|
39
38
|
|
@@ -93,9 +92,15 @@ module Sidekiq
|
|
93
92
|
end
|
94
93
|
|
95
94
|
def log_info(options)
|
96
|
-
# Don't log Redis AUTH password
|
97
95
|
redacted = "REDACTED"
|
98
|
-
|
96
|
+
|
97
|
+
# deep clone so we can muck with these options all we want
|
98
|
+
#
|
99
|
+
# exclude SSL params from dump-and-load because some information isn't
|
100
|
+
# safely dumpable in current Rubies
|
101
|
+
keys = options.keys
|
102
|
+
keys.delete(:ssl_params)
|
103
|
+
scrubbed_options = Marshal.load(Marshal.dump(options.slice(*keys)))
|
99
104
|
if scrubbed_options[:url] && (uri = URI.parse(scrubbed_options[:url])) && uri.password
|
100
105
|
uri.password = redacted
|
101
106
|
scrubbed_options[:url] = uri.to_s
|
@@ -103,6 +108,9 @@ module Sidekiq
|
|
103
108
|
if scrubbed_options[:password]
|
104
109
|
scrubbed_options[:password] = redacted
|
105
110
|
end
|
111
|
+
scrubbed_options[:sentinels]&.each do |sentinel|
|
112
|
+
sentinel[:password] = redacted if sentinel[:password]
|
113
|
+
end
|
106
114
|
if Sidekiq.server?
|
107
115
|
Sidekiq.logger.info("Booting Sidekiq #{Sidekiq::VERSION} with redis options #{scrubbed_options}")
|
108
116
|
else
|
@@ -119,7 +127,7 @@ module Sidekiq
|
|
119
127
|
# initialization code at all.
|
120
128
|
#
|
121
129
|
p = ENV["REDIS_PROVIDER"]
|
122
|
-
if p && p =~
|
130
|
+
if p && p =~ /:/
|
123
131
|
raise <<~EOM
|
124
132
|
REDIS_PROVIDER should be set to the name of the variable which contains the Redis URL, not a URL itself.
|
125
133
|
Platforms like Heroku will sell addons that publish a *_URL variable. You need to tell Sidekiq with REDIS_PROVIDER, e.g.:
|
data/lib/sidekiq/scheduled.rb
CHANGED
@@ -49,6 +49,7 @@ module Sidekiq
|
|
49
49
|
@sleeper = ConnectionPool::TimedStack.new
|
50
50
|
@done = false
|
51
51
|
@thread = nil
|
52
|
+
@count_calls = 0
|
52
53
|
end
|
53
54
|
|
54
55
|
# Shut down this instance, will pause until the thread is dead.
|
@@ -152,8 +153,13 @@ module Sidekiq
|
|
152
153
|
end
|
153
154
|
|
154
155
|
def process_count
|
155
|
-
|
156
|
+
# The work buried within Sidekiq::ProcessSet#cleanup can be
|
157
|
+
# expensive at scale. Cut it down by 90% with this counter.
|
158
|
+
# NB: This method is only called by the scheduler thread so we
|
159
|
+
# don't need to worry about the thread safety of +=.
|
160
|
+
pcount = Sidekiq::ProcessSet.new(@count_calls % 10 == 0).size
|
156
161
|
pcount = 1 if pcount == 0
|
162
|
+
@count_calls += 1
|
157
163
|
pcount
|
158
164
|
end
|
159
165
|
|
@@ -0,0 +1,149 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
# The MIT License
|
4
|
+
#
|
5
|
+
# Copyright (c) 2017, 2018, 2019, 2020 Agis Anastasopoulos
|
6
|
+
#
|
7
|
+
# Permission is hereby granted, free of charge, to any person obtaining a copy of
|
8
|
+
# this software and associated documentation files (the "Software"), to deal in
|
9
|
+
# the Software without restriction, including without limitation the rights to
|
10
|
+
# use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
|
11
|
+
# the Software, and to permit persons to whom the Software is furnished to do so,
|
12
|
+
# subject to the following conditions:
|
13
|
+
#
|
14
|
+
# The above copyright notice and this permission notice shall be included in all
|
15
|
+
# copies or substantial portions of the Software.
|
16
|
+
#
|
17
|
+
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
18
|
+
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
|
19
|
+
# FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
|
20
|
+
# COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
|
21
|
+
# IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
|
22
|
+
# CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
23
|
+
|
24
|
+
# This is a copy of https://github.com/agis/ruby-sdnotify as of commit a7d52ee
|
25
|
+
# The only changes made was "rehoming" it within the Sidekiq module to avoid
|
26
|
+
# namespace collisions and applying standard's code formatting style.
|
27
|
+
|
28
|
+
require "socket"
|
29
|
+
|
30
|
+
# SdNotify is a pure-Ruby implementation of sd_notify(3). It can be used to
|
31
|
+
# notify systemd about state changes. Methods of this package are no-op on
|
32
|
+
# non-systemd systems (eg. Darwin).
|
33
|
+
#
|
34
|
+
# The API maps closely to the original implementation of sd_notify(3),
|
35
|
+
# therefore be sure to check the official man pages prior to using SdNotify.
|
36
|
+
#
|
37
|
+
# @see https://www.freedesktop.org/software/systemd/man/sd_notify.html
|
38
|
+
module Sidekiq
|
39
|
+
module SdNotify
|
40
|
+
# Exception raised when there's an error writing to the notification socket
|
41
|
+
class NotifyError < RuntimeError; end
|
42
|
+
|
43
|
+
READY = "READY=1"
|
44
|
+
RELOADING = "RELOADING=1"
|
45
|
+
STOPPING = "STOPPING=1"
|
46
|
+
STATUS = "STATUS="
|
47
|
+
ERRNO = "ERRNO="
|
48
|
+
MAINPID = "MAINPID="
|
49
|
+
WATCHDOG = "WATCHDOG=1"
|
50
|
+
FDSTORE = "FDSTORE=1"
|
51
|
+
|
52
|
+
def self.ready(unset_env = false)
|
53
|
+
notify(READY, unset_env)
|
54
|
+
end
|
55
|
+
|
56
|
+
def self.reloading(unset_env = false)
|
57
|
+
notify(RELOADING, unset_env)
|
58
|
+
end
|
59
|
+
|
60
|
+
def self.stopping(unset_env = false)
|
61
|
+
notify(STOPPING, unset_env)
|
62
|
+
end
|
63
|
+
|
64
|
+
# @param status [String] a custom status string that describes the current
|
65
|
+
# state of the service
|
66
|
+
def self.status(status, unset_env = false)
|
67
|
+
notify("#{STATUS}#{status}", unset_env)
|
68
|
+
end
|
69
|
+
|
70
|
+
# @param errno [Integer]
|
71
|
+
def self.errno(errno, unset_env = false)
|
72
|
+
notify("#{ERRNO}#{errno}", unset_env)
|
73
|
+
end
|
74
|
+
|
75
|
+
# @param pid [Integer]
|
76
|
+
def self.mainpid(pid, unset_env = false)
|
77
|
+
notify("#{MAINPID}#{pid}", unset_env)
|
78
|
+
end
|
79
|
+
|
80
|
+
def self.watchdog(unset_env = false)
|
81
|
+
notify(WATCHDOG, unset_env)
|
82
|
+
end
|
83
|
+
|
84
|
+
def self.fdstore(unset_env = false)
|
85
|
+
notify(FDSTORE, unset_env)
|
86
|
+
end
|
87
|
+
|
88
|
+
# @return [Boolean] true if the service manager expects watchdog keep-alive
|
89
|
+
# notification messages to be sent from this process.
|
90
|
+
#
|
91
|
+
# If the $WATCHDOG_USEC environment variable is set,
|
92
|
+
# and the $WATCHDOG_PID variable is unset or set to the PID of the current
|
93
|
+
# process
|
94
|
+
#
|
95
|
+
# @note Unlike sd_watchdog_enabled(3), this method does not mutate the
|
96
|
+
# environment.
|
97
|
+
def self.watchdog?
|
98
|
+
wd_usec = ENV["WATCHDOG_USEC"]
|
99
|
+
wd_pid = ENV["WATCHDOG_PID"]
|
100
|
+
|
101
|
+
return false unless wd_usec
|
102
|
+
|
103
|
+
begin
|
104
|
+
wd_usec = Integer(wd_usec)
|
105
|
+
rescue
|
106
|
+
return false
|
107
|
+
end
|
108
|
+
|
109
|
+
return false if wd_usec <= 0
|
110
|
+
return true if !wd_pid || wd_pid == $$.to_s
|
111
|
+
|
112
|
+
false
|
113
|
+
end
|
114
|
+
|
115
|
+
# Notify systemd with the provided state, via the notification socket, if
|
116
|
+
# any.
|
117
|
+
#
|
118
|
+
# Generally this method will be used indirectly through the other methods
|
119
|
+
# of the library.
|
120
|
+
#
|
121
|
+
# @param state [String]
|
122
|
+
# @param unset_env [Boolean]
|
123
|
+
#
|
124
|
+
# @return [Fixnum, nil] the number of bytes written to the notification
|
125
|
+
# socket or nil if there was no socket to report to (eg. the program wasn't
|
126
|
+
# started by systemd)
|
127
|
+
#
|
128
|
+
# @raise [NotifyError] if there was an error communicating with the systemd
|
129
|
+
# socket
|
130
|
+
#
|
131
|
+
# @see https://www.freedesktop.org/software/systemd/man/sd_notify.html
|
132
|
+
def self.notify(state, unset_env = false)
|
133
|
+
sock = ENV["NOTIFY_SOCKET"]
|
134
|
+
|
135
|
+
return nil unless sock
|
136
|
+
|
137
|
+
ENV.delete("NOTIFY_SOCKET") if unset_env
|
138
|
+
|
139
|
+
begin
|
140
|
+
Addrinfo.unix(sock, :DGRAM).connect do |s|
|
141
|
+
s.close_on_exec = true
|
142
|
+
s.write(state)
|
143
|
+
end
|
144
|
+
rescue => e
|
145
|
+
raise NotifyError, "#{e.class}: #{e.message}", e.backtrace
|
146
|
+
end
|
147
|
+
end
|
148
|
+
end
|
149
|
+
end
|
@@ -0,0 +1,24 @@
|
|
1
|
+
#
|
2
|
+
# Sidekiq's systemd integration allows Sidekiq to inform systemd:
|
3
|
+
# 1. when it has successfully started
|
4
|
+
# 2. when it is starting shutdown
|
5
|
+
# 3. periodically for a liveness check with a watchdog thread
|
6
|
+
#
|
7
|
+
module Sidekiq
|
8
|
+
def self.start_watchdog
|
9
|
+
usec = Integer(ENV["WATCHDOG_USEC"])
|
10
|
+
return Sidekiq.logger.error("systemd Watchdog too fast: " + usec) if usec < 1_000_000
|
11
|
+
|
12
|
+
sec_f = usec / 1_000_000.0
|
13
|
+
# "It is recommended that a daemon sends a keep-alive notification message
|
14
|
+
# to the service manager every half of the time returned here."
|
15
|
+
ping_f = sec_f / 2
|
16
|
+
Sidekiq.logger.info "Pinging systemd watchdog every #{ping_f.round(1)} sec"
|
17
|
+
Thread.new do
|
18
|
+
loop do
|
19
|
+
sleep ping_f
|
20
|
+
Sidekiq::SdNotify.watchdog
|
21
|
+
end
|
22
|
+
end
|
23
|
+
end
|
24
|
+
end
|
data/lib/sidekiq/testing.rb
CHANGED
@@ -337,8 +337,6 @@ module Sidekiq
|
|
337
337
|
Sidekiq::Extensions::DelayedModel.extend(TestingExtensions) if defined?(Sidekiq::Extensions::DelayedModel)
|
338
338
|
end
|
339
339
|
|
340
|
-
if defined?(::Rails) && Rails.respond_to?(:env) && !Rails.env.test?
|
341
|
-
|
342
|
-
puts("⛔️ WARNING: Sidekiq testing API enabled, but this is not the test environment. Your jobs will not go to Redis.")
|
343
|
-
puts("**************************************************")
|
340
|
+
if defined?(::Rails) && Rails.respond_to?(:env) && !Rails.env.test? && !$TESTING
|
341
|
+
warn("⛔️ WARNING: Sidekiq testing API enabled, but this is not the test environment. Your jobs will not go to Redis.", uplevel: 1)
|
344
342
|
end
|
data/lib/sidekiq/util.rb
CHANGED
@@ -1,5 +1,6 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
+
require "forwardable"
|
3
4
|
require "socket"
|
4
5
|
require "securerandom"
|
5
6
|
require "sidekiq/exception_handler"
|
@@ -8,11 +9,36 @@ module Sidekiq
|
|
8
9
|
##
|
9
10
|
# This module is part of Sidekiq core and not intended for extensions.
|
10
11
|
#
|
12
|
+
|
13
|
+
class RingBuffer
|
14
|
+
include Enumerable
|
15
|
+
extend Forwardable
|
16
|
+
def_delegators :@buf, :[], :each, :size
|
17
|
+
|
18
|
+
def initialize(size, default = 0)
|
19
|
+
@size = size
|
20
|
+
@buf = Array.new(size, default)
|
21
|
+
@index = 0
|
22
|
+
end
|
23
|
+
|
24
|
+
def <<(element)
|
25
|
+
@buf[@index % @size] = element
|
26
|
+
@index += 1
|
27
|
+
element
|
28
|
+
end
|
29
|
+
|
30
|
+
def buffer
|
31
|
+
@buf
|
32
|
+
end
|
33
|
+
|
34
|
+
def reset(default = 0)
|
35
|
+
@buf.fill(default)
|
36
|
+
end
|
37
|
+
end
|
38
|
+
|
11
39
|
module Util
|
12
40
|
include ExceptionHandler
|
13
41
|
|
14
|
-
EXPIRY = 60 * 60 * 24
|
15
|
-
|
16
42
|
def watchdog(last_words)
|
17
43
|
yield
|
18
44
|
rescue Exception => ex
|
data/lib/sidekiq/version.rb
CHANGED
data/lib/sidekiq/web/action.rb
CHANGED
@@ -15,7 +15,7 @@ module Sidekiq
|
|
15
15
|
end
|
16
16
|
|
17
17
|
def halt(res)
|
18
|
-
throw :halt, res
|
18
|
+
throw :halt, [res, {"Content-Type" => "text/plain"}, [res.to_s]]
|
19
19
|
end
|
20
20
|
|
21
21
|
def redirect(location)
|
@@ -68,7 +68,7 @@ module Sidekiq
|
|
68
68
|
end
|
69
69
|
|
70
70
|
def json(payload)
|
71
|
-
[200, {"Content-Type" => "application/json", "Cache-Control" => "no-
|
71
|
+
[200, {"Content-Type" => "application/json", "Cache-Control" => "private, no-store"}, [Sidekiq.dump_json(payload)]]
|
72
72
|
end
|
73
73
|
|
74
74
|
def initialize(env, block)
|
@@ -4,8 +4,6 @@ module Sidekiq
|
|
4
4
|
class WebApplication
|
5
5
|
extend WebRouter
|
6
6
|
|
7
|
-
CONTENT_LENGTH = "Content-Length"
|
8
|
-
CONTENT_TYPE = "Content-Type"
|
9
7
|
REDIS_KEYS = %w[redis_version uptime_in_days connected_clients used_memory_human used_memory_peak_human]
|
10
8
|
CSP_HEADER = [
|
11
9
|
"default-src 'self' https: http:",
|
@@ -20,7 +18,7 @@ module Sidekiq
|
|
20
18
|
"script-src 'self' https: http: 'unsafe-inline'",
|
21
19
|
"style-src 'self' https: http: 'unsafe-inline'",
|
22
20
|
"worker-src 'self'",
|
23
|
-
"base-uri 'self'"
|
21
|
+
"base-uri 'self'"
|
24
22
|
].join("; ").freeze
|
25
23
|
|
26
24
|
def initialize(klass)
|
@@ -43,6 +41,13 @@ module Sidekiq
|
|
43
41
|
# nothing, backwards compatibility
|
44
42
|
end
|
45
43
|
|
44
|
+
head "/" do
|
45
|
+
# HEAD / is the cheapest heartbeat possible,
|
46
|
+
# it hits Redis to ensure connectivity
|
47
|
+
Sidekiq.redis { |c| c.llen("queue:default") }
|
48
|
+
""
|
49
|
+
end
|
50
|
+
|
46
51
|
get "/" do
|
47
52
|
@redis_info = redis_info.select { |k, v| REDIS_KEYS.include? k }
|
48
53
|
stats_history = Sidekiq::Stats::History.new((params["days"] || 30).to_i)
|
@@ -77,28 +82,38 @@ module Sidekiq
|
|
77
82
|
erb(:queues)
|
78
83
|
end
|
79
84
|
|
85
|
+
QUEUE_NAME = /\A[a-z_:.\-0-9]+\z/i
|
86
|
+
|
80
87
|
get "/queues/:name" do
|
81
88
|
@name = route_params[:name]
|
82
89
|
|
83
|
-
halt(404)
|
90
|
+
halt(404) if !@name || @name !~ QUEUE_NAME
|
84
91
|
|
85
92
|
@count = (params["count"] || 25).to_i
|
86
93
|
@queue = Sidekiq::Queue.new(@name)
|
87
94
|
(@current_page, @total_size, @messages) = page("queue:#{@name}", params["page"], @count, reverse: params["direction"] == "asc")
|
88
|
-
@messages = @messages.map { |msg| Sidekiq::
|
95
|
+
@messages = @messages.map { |msg| Sidekiq::JobRecord.new(msg, @name) }
|
89
96
|
|
90
97
|
erb(:queue)
|
91
98
|
end
|
92
99
|
|
93
100
|
post "/queues/:name" do
|
94
|
-
Sidekiq::Queue.new(route_params[:name])
|
101
|
+
queue = Sidekiq::Queue.new(route_params[:name])
|
102
|
+
|
103
|
+
if Sidekiq.pro? && params["pause"]
|
104
|
+
queue.pause!
|
105
|
+
elsif Sidekiq.pro? && params["unpause"]
|
106
|
+
queue.unpause!
|
107
|
+
else
|
108
|
+
queue.clear
|
109
|
+
end
|
95
110
|
|
96
111
|
redirect "#{root_path}queues"
|
97
112
|
end
|
98
113
|
|
99
114
|
post "/queues/:name/delete" do
|
100
115
|
name = route_params[:name]
|
101
|
-
Sidekiq::
|
116
|
+
Sidekiq::JobRecord.new(params["key_val"], name).delete
|
102
117
|
|
103
118
|
redirect_with_query("#{root_path}queues/#{CGI.escape(name)}")
|
104
119
|
end
|
@@ -268,7 +283,7 @@ module Sidekiq
|
|
268
283
|
scheduled: sidekiq_stats.scheduled_size,
|
269
284
|
retries: sidekiq_stats.retry_size,
|
270
285
|
dead: sidekiq_stats.dead_size,
|
271
|
-
default_latency: sidekiq_stats.default_queue_latency
|
286
|
+
default_latency: sidekiq_stats.default_queue_latency
|
272
287
|
},
|
273
288
|
redis: redis_stats,
|
274
289
|
server_utc_time: server_utc_time
|
@@ -291,29 +306,25 @@ module Sidekiq
|
|
291
306
|
self.class.run_afters(app, action)
|
292
307
|
end
|
293
308
|
|
294
|
-
|
309
|
+
case resp
|
295
310
|
when Array
|
311
|
+
# redirects go here
|
296
312
|
resp
|
297
313
|
else
|
314
|
+
# rendered content goes here
|
298
315
|
headers = {
|
299
316
|
"Content-Type" => "text/html",
|
300
|
-
"Cache-Control" => "no-
|
317
|
+
"Cache-Control" => "private, no-store",
|
301
318
|
"Content-Language" => action.locale,
|
302
|
-
"Content-Security-Policy" => CSP_HEADER
|
319
|
+
"Content-Security-Policy" => CSP_HEADER
|
303
320
|
}
|
304
|
-
|
321
|
+
# we'll let Rack calculate Content-Length for us.
|
305
322
|
[200, headers, [resp]]
|
306
323
|
end
|
307
|
-
|
308
|
-
resp[1] = resp[1].dup
|
309
|
-
|
310
|
-
resp[1][CONTENT_LENGTH] = resp[2].inject(0) { |l, p| l + p.bytesize }.to_s
|
311
|
-
|
312
|
-
resp
|
313
324
|
end
|
314
325
|
|
315
326
|
def self.helpers(mod = nil, &block)
|
316
|
-
if
|
327
|
+
if block
|
317
328
|
WebAction.class_eval(&block)
|
318
329
|
else
|
319
330
|
WebAction.send(:include, mod)
|