sidekiq 6.0.0 → 6.5.7
Sign up to get free protection for your applications and to get access to all the features.
Potentially problematic release.
This version of sidekiq might be problematic. Click here for more details.
- checksums.yaml +4 -4
- data/Changes.md +383 -2
- data/LICENSE +3 -3
- data/README.md +13 -10
- data/bin/sidekiq +27 -3
- data/bin/sidekiqload +74 -66
- data/bin/sidekiqmon +5 -6
- data/lib/generators/sidekiq/job_generator.rb +57 -0
- data/lib/generators/sidekiq/templates/{worker.rb.erb → job.rb.erb} +2 -2
- data/lib/generators/sidekiq/templates/{worker_spec.rb.erb → job_spec.rb.erb} +1 -1
- data/lib/generators/sidekiq/templates/{worker_test.rb.erb → job_test.rb.erb} +1 -1
- data/lib/sidekiq/api.rb +446 -221
- data/lib/sidekiq/cli.rb +112 -63
- data/lib/sidekiq/client.rb +57 -60
- data/lib/sidekiq/{util.rb → component.rb} +12 -16
- data/lib/sidekiq/delay.rb +3 -1
- data/lib/sidekiq/extensions/action_mailer.rb +3 -2
- data/lib/sidekiq/extensions/active_record.rb +4 -3
- data/lib/sidekiq/extensions/class_methods.rb +5 -4
- data/lib/sidekiq/extensions/generic_proxy.rb +4 -2
- data/lib/sidekiq/fetch.rb +48 -37
- data/lib/sidekiq/job.rb +13 -0
- data/lib/sidekiq/job_logger.rb +19 -23
- data/lib/sidekiq/job_retry.rb +100 -67
- data/lib/sidekiq/job_util.rb +71 -0
- data/lib/sidekiq/launcher.rb +145 -59
- data/lib/sidekiq/logger.rb +99 -12
- data/lib/sidekiq/manager.rb +35 -34
- data/lib/sidekiq/metrics/deploy.rb +47 -0
- data/lib/sidekiq/metrics/query.rb +153 -0
- data/lib/sidekiq/metrics/shared.rb +94 -0
- data/lib/sidekiq/metrics/tracking.rb +134 -0
- data/lib/sidekiq/middleware/chain.rb +99 -44
- data/lib/sidekiq/middleware/current_attributes.rb +63 -0
- data/lib/sidekiq/middleware/i18n.rb +6 -4
- data/lib/sidekiq/middleware/modules.rb +21 -0
- data/lib/sidekiq/monitor.rb +4 -19
- data/lib/sidekiq/paginator.rb +13 -8
- data/lib/sidekiq/processor.rb +64 -60
- data/lib/sidekiq/rails.rb +38 -22
- data/lib/sidekiq/redis_client_adapter.rb +154 -0
- data/lib/sidekiq/redis_connection.rb +91 -54
- data/lib/sidekiq/ring_buffer.rb +29 -0
- data/lib/sidekiq/scheduled.rb +93 -28
- data/lib/sidekiq/sd_notify.rb +149 -0
- data/lib/sidekiq/systemd.rb +24 -0
- data/lib/sidekiq/testing/inline.rb +4 -4
- data/lib/sidekiq/testing.rb +51 -40
- data/lib/sidekiq/transaction_aware_client.rb +45 -0
- data/lib/sidekiq/version.rb +1 -1
- data/lib/sidekiq/web/action.rb +3 -3
- data/lib/sidekiq/web/application.rb +57 -34
- data/lib/sidekiq/web/csrf_protection.rb +180 -0
- data/lib/sidekiq/web/helpers.rb +77 -36
- data/lib/sidekiq/web/router.rb +6 -5
- data/lib/sidekiq/web.rb +41 -73
- data/lib/sidekiq/worker.rb +144 -21
- data/lib/sidekiq.rb +129 -32
- data/sidekiq.gemspec +14 -7
- data/web/assets/images/apple-touch-icon.png +0 -0
- data/web/assets/javascripts/application.js +112 -61
- data/web/assets/javascripts/chart.min.js +13 -0
- data/web/assets/javascripts/chartjs-plugin-annotation.min.js +7 -0
- data/web/assets/javascripts/dashboard.js +52 -69
- data/web/assets/javascripts/graph.js +16 -0
- data/web/assets/javascripts/metrics.js +262 -0
- data/web/assets/stylesheets/application-dark.css +143 -0
- data/web/assets/stylesheets/application-rtl.css +0 -4
- data/web/assets/stylesheets/application.css +88 -233
- data/web/locales/ar.yml +8 -2
- data/web/locales/de.yml +14 -2
- data/web/locales/el.yml +43 -19
- data/web/locales/en.yml +13 -1
- data/web/locales/es.yml +18 -2
- data/web/locales/fr.yml +10 -3
- data/web/locales/ja.yml +12 -0
- data/web/locales/lt.yml +83 -0
- data/web/locales/pl.yml +4 -4
- data/web/locales/pt-br.yml +27 -9
- data/web/locales/ru.yml +4 -0
- data/web/locales/vi.yml +83 -0
- data/web/locales/zh-cn.yml +36 -11
- data/web/locales/zh-tw.yml +32 -7
- data/web/views/_footer.erb +1 -1
- data/web/views/_job_info.erb +3 -2
- data/web/views/_nav.erb +1 -1
- data/web/views/_poll_link.erb +2 -5
- data/web/views/_summary.erb +7 -7
- data/web/views/busy.erb +56 -22
- data/web/views/dashboard.erb +23 -14
- data/web/views/dead.erb +3 -3
- data/web/views/layout.erb +3 -1
- data/web/views/metrics.erb +69 -0
- data/web/views/metrics_for_job.erb +87 -0
- data/web/views/morgue.erb +9 -6
- data/web/views/queue.erb +23 -10
- data/web/views/queues.erb +10 -2
- data/web/views/retries.erb +11 -8
- data/web/views/retry.erb +3 -3
- data/web/views/scheduled.erb +5 -2
- metadata +57 -58
- data/.circleci/config.yml +0 -61
- data/.github/contributing.md +0 -32
- data/.github/issue_template.md +0 -11
- data/.gitignore +0 -13
- data/.standard.yml +0 -20
- data/3.0-Upgrade.md +0 -70
- data/4.0-Upgrade.md +0 -53
- data/5.0-Upgrade.md +0 -56
- data/6.0-Upgrade.md +0 -70
- data/COMM-LICENSE +0 -97
- data/Ent-2.0-Upgrade.md +0 -37
- data/Ent-Changes.md +0 -250
- data/Gemfile +0 -24
- data/Gemfile.lock +0 -196
- data/Pro-2.0-Upgrade.md +0 -138
- data/Pro-3.0-Upgrade.md +0 -44
- data/Pro-4.0-Upgrade.md +0 -35
- data/Pro-5.0-Upgrade.md +0 -25
- data/Pro-Changes.md +0 -768
- data/Rakefile +0 -10
- data/code_of_conduct.md +0 -50
- data/lib/generators/sidekiq/worker_generator.rb +0 -47
- data/lib/sidekiq/exception_handler.rb +0 -27
@@ -10,16 +10,18 @@ module Sidekiq::Middleware::I18n
|
|
10
10
|
# Get the current locale and store it in the message
|
11
11
|
# to be sent to Sidekiq.
|
12
12
|
class Client
|
13
|
-
|
14
|
-
|
13
|
+
include Sidekiq::ClientMiddleware
|
14
|
+
def call(_jobclass, job, _queue, _redis)
|
15
|
+
job["locale"] ||= I18n.locale
|
15
16
|
yield
|
16
17
|
end
|
17
18
|
end
|
18
19
|
|
19
20
|
# Pull the msg locale out and set the current thread to use it.
|
20
21
|
class Server
|
21
|
-
|
22
|
-
|
22
|
+
include Sidekiq::ServerMiddleware
|
23
|
+
def call(_jobclass, job, _queue, &block)
|
24
|
+
I18n.with_locale(job.fetch("locale", I18n.default_locale), &block)
|
23
25
|
end
|
24
26
|
end
|
25
27
|
end
|
@@ -0,0 +1,21 @@
|
|
1
|
+
module Sidekiq
|
2
|
+
# Server-side middleware must import this Module in order
|
3
|
+
# to get access to server resources during `call`.
|
4
|
+
module ServerMiddleware
|
5
|
+
attr_accessor :config
|
6
|
+
def redis_pool
|
7
|
+
config.redis_pool
|
8
|
+
end
|
9
|
+
|
10
|
+
def logger
|
11
|
+
config.logger
|
12
|
+
end
|
13
|
+
|
14
|
+
def redis(&block)
|
15
|
+
config.redis(&block)
|
16
|
+
end
|
17
|
+
end
|
18
|
+
|
19
|
+
# no difference for now
|
20
|
+
ClientMiddleware = ServerMiddleware
|
21
|
+
end
|
data/lib/sidekiq/monitor.rb
CHANGED
@@ -4,21 +4,6 @@ require "fileutils"
|
|
4
4
|
require "sidekiq/api"
|
5
5
|
|
6
6
|
class Sidekiq::Monitor
|
7
|
-
CMD = File.basename($PROGRAM_NAME)
|
8
|
-
|
9
|
-
attr_reader :stage
|
10
|
-
|
11
|
-
def self.print_usage
|
12
|
-
puts "#{CMD} - monitor Sidekiq from the command line."
|
13
|
-
puts
|
14
|
-
puts "Usage: #{CMD} status <section>"
|
15
|
-
puts
|
16
|
-
puts " <section> (optional) view a specific section of the status output"
|
17
|
-
puts " Valid sections are: #{Sidekiq::Monitor::Status::VALID_SECTIONS.join(", ")}"
|
18
|
-
puts
|
19
|
-
puts "Set REDIS_URL to the location of your Redis server if not monitoring localhost."
|
20
|
-
end
|
21
|
-
|
22
7
|
class Status
|
23
8
|
VALID_SECTIONS = %w[all version overview processes queues]
|
24
9
|
COL_PAD = 2
|
@@ -32,7 +17,7 @@ class Sidekiq::Monitor
|
|
32
17
|
end
|
33
18
|
send(section)
|
34
19
|
rescue => e
|
35
|
-
|
20
|
+
abort "Couldn't get status: #{e}"
|
36
21
|
end
|
37
22
|
|
38
23
|
def all
|
@@ -47,7 +32,7 @@ class Sidekiq::Monitor
|
|
47
32
|
|
48
33
|
def version
|
49
34
|
puts "Sidekiq #{Sidekiq::VERSION}"
|
50
|
-
puts Time.now
|
35
|
+
puts Time.now.utc
|
51
36
|
end
|
52
37
|
|
53
38
|
def overview
|
@@ -77,7 +62,7 @@ class Sidekiq::Monitor
|
|
77
62
|
columns = {
|
78
63
|
name: [:ljust, (["name"] + queue_data.map(&:name)).map(&:length).max + COL_PAD],
|
79
64
|
size: [:rjust, (["size"] + queue_data.map(&:size)).map(&:length).max + COL_PAD],
|
80
|
-
latency: [:rjust, (["latency"] + queue_data.map(&:latency)).map(&:length).max + COL_PAD]
|
65
|
+
latency: [:rjust, (["latency"] + queue_data.map(&:latency)).map(&:length).max + COL_PAD]
|
81
66
|
}
|
82
67
|
columns.each { |col, (dir, width)| print col.to_s.upcase.public_send(dir, width) }
|
83
68
|
puts
|
@@ -116,7 +101,7 @@ class Sidekiq::Monitor
|
|
116
101
|
tags = [
|
117
102
|
process["tag"],
|
118
103
|
process["labels"],
|
119
|
-
(process["quiet"] == "true" ? "quiet" : nil)
|
104
|
+
(process["quiet"] == "true" ? "quiet" : nil)
|
120
105
|
].flatten.compact
|
121
106
|
tags.any? ? "[#{tags.join("] [")}]" : nil
|
122
107
|
end
|
data/lib/sidekiq/paginator.rb
CHANGED
@@ -12,24 +12,29 @@ module Sidekiq
|
|
12
12
|
|
13
13
|
Sidekiq.redis do |conn|
|
14
14
|
type = conn.type(key)
|
15
|
+
rev = opts && opts[:reverse]
|
15
16
|
|
16
17
|
case type
|
17
18
|
when "zset"
|
18
|
-
|
19
|
-
|
20
|
-
conn.zcard(key)
|
19
|
+
total_size, items = conn.multi { |transaction|
|
20
|
+
transaction.zcard(key)
|
21
21
|
if rev
|
22
|
-
|
22
|
+
transaction.zrevrange(key, starting, ending, withscores: true)
|
23
23
|
else
|
24
|
-
|
24
|
+
transaction.zrange(key, starting, ending, withscores: true)
|
25
25
|
end
|
26
26
|
}
|
27
27
|
[current_page, total_size, items]
|
28
28
|
when "list"
|
29
|
-
total_size, items = conn.multi {
|
30
|
-
|
31
|
-
|
29
|
+
total_size, items = conn.multi { |transaction|
|
30
|
+
transaction.llen(key)
|
31
|
+
if rev
|
32
|
+
transaction.lrange(key, -ending - 1, -starting - 1)
|
33
|
+
else
|
34
|
+
transaction.lrange(key, starting, ending)
|
35
|
+
end
|
32
36
|
}
|
37
|
+
items.reverse! if rev
|
33
38
|
[current_page, total_size, items]
|
34
39
|
when "none"
|
35
40
|
[1, 0, []]
|
data/lib/sidekiq/processor.rb
CHANGED
@@ -1,6 +1,5 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
-
require "sidekiq/util"
|
4
3
|
require "sidekiq/fetch"
|
5
4
|
require "sidekiq/job_logger"
|
6
5
|
require "sidekiq/job_retry"
|
@@ -11,33 +10,34 @@ module Sidekiq
|
|
11
10
|
#
|
12
11
|
# 1. fetches a job from Redis
|
13
12
|
# 2. executes the job
|
14
|
-
# a. instantiate the
|
13
|
+
# a. instantiate the job class
|
15
14
|
# b. run the middleware chain
|
16
15
|
# c. call #perform
|
17
16
|
#
|
18
|
-
# A Processor can exit due to shutdown
|
19
|
-
#
|
17
|
+
# A Processor can exit due to shutdown or due to
|
18
|
+
# an error during job execution.
|
20
19
|
#
|
21
20
|
# If an error occurs in the job execution, the
|
22
21
|
# Processor calls the Manager to create a new one
|
23
22
|
# to replace itself and exits.
|
24
23
|
#
|
25
24
|
class Processor
|
26
|
-
include
|
25
|
+
include Sidekiq::Component
|
27
26
|
|
28
27
|
attr_reader :thread
|
29
28
|
attr_reader :job
|
30
29
|
|
31
|
-
def initialize(
|
32
|
-
@
|
30
|
+
def initialize(options, &block)
|
31
|
+
@callback = block
|
33
32
|
@down = false
|
34
33
|
@done = false
|
35
34
|
@job = nil
|
36
35
|
@thread = nil
|
37
|
-
@
|
38
|
-
@
|
39
|
-
@
|
40
|
-
@
|
36
|
+
@config = options
|
37
|
+
@strategy = options[:fetch]
|
38
|
+
@reloader = options[:reloader] || proc { |&block| block.call }
|
39
|
+
@job_logger = (options[:job_logger] || Sidekiq::JobLogger).new
|
40
|
+
@retrier = Sidekiq::JobRetry.new(options)
|
41
41
|
end
|
42
42
|
|
43
43
|
def terminate(wait = false)
|
@@ -66,26 +66,26 @@ module Sidekiq
|
|
66
66
|
|
67
67
|
def run
|
68
68
|
process_one until @done
|
69
|
-
@
|
69
|
+
@callback.call(self)
|
70
70
|
rescue Sidekiq::Shutdown
|
71
|
-
@
|
71
|
+
@callback.call(self)
|
72
72
|
rescue Exception => ex
|
73
|
-
@
|
73
|
+
@callback.call(self, ex)
|
74
74
|
end
|
75
75
|
|
76
|
-
def process_one
|
76
|
+
def process_one(&block)
|
77
77
|
@job = fetch
|
78
78
|
process(@job) if @job
|
79
79
|
@job = nil
|
80
80
|
end
|
81
81
|
|
82
82
|
def get_one
|
83
|
-
|
83
|
+
uow = @strategy.retrieve_work
|
84
84
|
if @down
|
85
85
|
logger.info { "Redis is online, #{::Process.clock_gettime(::Process::CLOCK_MONOTONIC) - @down} sec downtime" }
|
86
86
|
@down = nil
|
87
87
|
end
|
88
|
-
|
88
|
+
uow
|
89
89
|
rescue Sidekiq::Shutdown
|
90
90
|
rescue => ex
|
91
91
|
handle_fetch_exception(ex)
|
@@ -111,26 +111,29 @@ module Sidekiq
|
|
111
111
|
nil
|
112
112
|
end
|
113
113
|
|
114
|
-
def dispatch(job_hash, queue)
|
114
|
+
def dispatch(job_hash, queue, jobstr)
|
115
115
|
# since middleware can mutate the job hash
|
116
|
-
# we clone
|
116
|
+
# we need to clone it to report the original
|
117
117
|
# job structure to the Web UI
|
118
|
-
|
118
|
+
# or to push back to redis when retrying.
|
119
|
+
# To avoid costly and, most of the time, useless cloning here,
|
120
|
+
# we pass original String of JSON to respected methods
|
121
|
+
# to re-parse it there if we need access to the original, untouched job
|
119
122
|
|
120
|
-
@job_logger.
|
121
|
-
@retrier.global(
|
123
|
+
@job_logger.prepare(job_hash) do
|
124
|
+
@retrier.global(jobstr, queue) do
|
122
125
|
@job_logger.call(job_hash, queue) do
|
123
|
-
stats(
|
126
|
+
stats(jobstr, queue) do
|
124
127
|
# Rails 5 requires a Reloader to wrap code execution. In order to
|
125
128
|
# constantize the worker and instantiate an instance, we have to call
|
126
129
|
# the Reloader. It handles code loading, db connection management, etc.
|
127
130
|
# Effectively this block denotes a "unit of work" to Rails.
|
128
131
|
@reloader.call do
|
129
132
|
klass = constantize(job_hash["class"])
|
130
|
-
|
131
|
-
|
132
|
-
@retrier.local(
|
133
|
-
yield
|
133
|
+
inst = klass.new
|
134
|
+
inst.jid = job_hash["jid"]
|
135
|
+
@retrier.local(inst, jobstr, queue) do
|
136
|
+
yield inst
|
134
137
|
end
|
135
138
|
end
|
136
139
|
end
|
@@ -139,9 +142,9 @@ module Sidekiq
|
|
139
142
|
end
|
140
143
|
end
|
141
144
|
|
142
|
-
def process(
|
143
|
-
jobstr =
|
144
|
-
queue =
|
145
|
+
def process(uow)
|
146
|
+
jobstr = uow.job
|
147
|
+
queue = uow.queue_name
|
145
148
|
|
146
149
|
# Treat malformed JSON as a special case: job goes straight to the morgue.
|
147
150
|
job_hash = nil
|
@@ -149,16 +152,22 @@ module Sidekiq
|
|
149
152
|
job_hash = Sidekiq.load_json(jobstr)
|
150
153
|
rescue => ex
|
151
154
|
handle_exception(ex, {context: "Invalid JSON for job", jobstr: jobstr})
|
152
|
-
|
153
|
-
|
154
|
-
|
155
|
+
now = Time.now.to_f
|
156
|
+
config.redis do |conn|
|
157
|
+
conn.multi do |xa|
|
158
|
+
xa.zadd("dead", now.to_s, jobstr)
|
159
|
+
xa.zremrangebyscore("dead", "-inf", now - config[:dead_timeout_in_seconds])
|
160
|
+
xa.zremrangebyrank("dead", 0, - config[:dead_max_jobs])
|
161
|
+
end
|
162
|
+
end
|
163
|
+
return uow.acknowledge
|
155
164
|
end
|
156
165
|
|
157
166
|
ack = false
|
158
167
|
begin
|
159
|
-
dispatch(job_hash, queue) do |
|
160
|
-
|
161
|
-
execute_job(
|
168
|
+
dispatch(job_hash, queue, jobstr) do |inst|
|
169
|
+
@config.server_middleware.invoke(inst, job_hash, queue) do
|
170
|
+
execute_job(inst, job_hash["args"])
|
162
171
|
end
|
163
172
|
end
|
164
173
|
ack = true
|
@@ -171,26 +180,26 @@ module Sidekiq
|
|
171
180
|
# signals that we created a retry successfully. We can acknowlege the job.
|
172
181
|
ack = true
|
173
182
|
e = h.cause || h
|
174
|
-
handle_exception(e, {context: "Job raised exception", job: job_hash
|
183
|
+
handle_exception(e, {context: "Job raised exception", job: job_hash})
|
175
184
|
raise e
|
176
185
|
rescue Exception => ex
|
177
186
|
# Unexpected error! This is very bad and indicates an exception that got past
|
178
187
|
# the retry subsystem (e.g. network partition). We won't acknowledge the job
|
179
188
|
# so it can be rescued when using Sidekiq Pro.
|
180
189
|
handle_exception(ex, {context: "Internal exception!", job: job_hash, jobstr: jobstr})
|
181
|
-
raise
|
190
|
+
raise ex
|
182
191
|
ensure
|
183
192
|
if ack
|
184
193
|
# We don't want a shutdown signal to interrupt job acknowledgment.
|
185
194
|
Thread.handle_interrupt(Sidekiq::Shutdown => :never) do
|
186
|
-
|
195
|
+
uow.acknowledge
|
187
196
|
end
|
188
197
|
end
|
189
198
|
end
|
190
199
|
end
|
191
200
|
|
192
|
-
def execute_job(
|
193
|
-
|
201
|
+
def execute_job(inst, cloned_args)
|
202
|
+
inst.perform(*cloned_args)
|
194
203
|
end
|
195
204
|
|
196
205
|
# Ruby doesn't provide atomic counters out of the box so we'll
|
@@ -216,39 +225,39 @@ module Sidekiq
|
|
216
225
|
end
|
217
226
|
|
218
227
|
# jruby's Hash implementation is not threadsafe, so we wrap it in a mutex here
|
219
|
-
class
|
228
|
+
class SharedWorkState
|
220
229
|
def initialize
|
221
|
-
@
|
230
|
+
@work_state = {}
|
222
231
|
@lock = Mutex.new
|
223
232
|
end
|
224
233
|
|
225
234
|
def set(tid, hash)
|
226
|
-
@lock.synchronize { @
|
235
|
+
@lock.synchronize { @work_state[tid] = hash }
|
227
236
|
end
|
228
237
|
|
229
238
|
def delete(tid)
|
230
|
-
@lock.synchronize { @
|
239
|
+
@lock.synchronize { @work_state.delete(tid) }
|
231
240
|
end
|
232
241
|
|
233
242
|
def dup
|
234
|
-
@lock.synchronize { @
|
243
|
+
@lock.synchronize { @work_state.dup }
|
235
244
|
end
|
236
245
|
|
237
246
|
def size
|
238
|
-
@lock.synchronize { @
|
247
|
+
@lock.synchronize { @work_state.size }
|
239
248
|
end
|
240
249
|
|
241
250
|
def clear
|
242
|
-
@lock.synchronize { @
|
251
|
+
@lock.synchronize { @work_state.clear }
|
243
252
|
end
|
244
253
|
end
|
245
254
|
|
246
255
|
PROCESSED = Counter.new
|
247
256
|
FAILURE = Counter.new
|
248
|
-
|
257
|
+
WORK_STATE = SharedWorkState.new
|
249
258
|
|
250
|
-
def stats(
|
251
|
-
|
259
|
+
def stats(jobstr, queue)
|
260
|
+
WORK_STATE.set(tid, {queue: queue, payload: jobstr, run_at: Time.now.to_i})
|
252
261
|
|
253
262
|
begin
|
254
263
|
yield
|
@@ -256,26 +265,21 @@ module Sidekiq
|
|
256
265
|
FAILURE.incr
|
257
266
|
raise
|
258
267
|
ensure
|
259
|
-
|
268
|
+
WORK_STATE.delete(tid)
|
260
269
|
PROCESSED.incr
|
261
270
|
end
|
262
271
|
end
|
263
272
|
|
264
|
-
# Deep clone the arguments passed to the worker so that if
|
265
|
-
# the job fails, what is pushed back onto Redis hasn't
|
266
|
-
# been mutated by the worker.
|
267
|
-
def cloned(thing)
|
268
|
-
Marshal.load(Marshal.dump(thing))
|
269
|
-
end
|
270
|
-
|
271
273
|
def constantize(str)
|
274
|
+
return Object.const_get(str) unless str.include?("::")
|
275
|
+
|
272
276
|
names = str.split("::")
|
273
277
|
names.shift if names.empty? || names.first.empty?
|
274
278
|
|
275
279
|
names.inject(Object) do |constant, name|
|
276
280
|
# the false flag limits search for name to under the constant namespace
|
277
281
|
# which mimics Rails' behaviour
|
278
|
-
constant.
|
282
|
+
constant.const_get(name, false)
|
279
283
|
end
|
280
284
|
end
|
281
285
|
end
|
data/lib/sidekiq/rails.rb
CHANGED
@@ -1,9 +1,25 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
-
require "sidekiq/
|
3
|
+
require "sidekiq/job"
|
4
4
|
|
5
5
|
module Sidekiq
|
6
6
|
class Rails < ::Rails::Engine
|
7
|
+
class Reloader
|
8
|
+
def initialize(app = ::Rails.application)
|
9
|
+
@app = app
|
10
|
+
end
|
11
|
+
|
12
|
+
def call
|
13
|
+
@app.reloader.wrap do
|
14
|
+
yield
|
15
|
+
end
|
16
|
+
end
|
17
|
+
|
18
|
+
def inspect
|
19
|
+
"#<Sidekiq::Rails::Reloader @app=#{@app.class.name}>"
|
20
|
+
end
|
21
|
+
end
|
22
|
+
|
7
23
|
# By including the Options module, we allow AJs to directly control sidekiq features
|
8
24
|
# via the *sidekiq_options* class method and, for instance, not use AJ's retry system.
|
9
25
|
# AJ retries don't show up in the Sidekiq UI Retries tab, save any error data, can't be
|
@@ -17,35 +33,35 @@ module Sidekiq
|
|
17
33
|
# end
|
18
34
|
initializer "sidekiq.active_job_integration" do
|
19
35
|
ActiveSupport.on_load(:active_job) do
|
20
|
-
include ::Sidekiq::
|
36
|
+
include ::Sidekiq::Job::Options unless respond_to?(:sidekiq_options)
|
37
|
+
end
|
38
|
+
end
|
39
|
+
|
40
|
+
initializer "sidekiq.rails_logger" do
|
41
|
+
Sidekiq.configure_server do |config|
|
42
|
+
# This is the integration code necessary so that if a job uses `Rails.logger.info "Hello"`,
|
43
|
+
# it will appear in the Sidekiq console with all of the job context. See #5021 and
|
44
|
+
# https://github.com/rails/rails/blob/b5f2b550f69a99336482739000c58e4e04e033aa/railties/lib/rails/commands/server/server_command.rb#L82-L84
|
45
|
+
unless ::Rails.logger == config.logger || ::ActiveSupport::Logger.logger_outputs_to?(::Rails.logger, $stdout)
|
46
|
+
::Rails.logger.extend(::ActiveSupport::Logger.broadcast(config.logger))
|
47
|
+
end
|
21
48
|
end
|
22
49
|
end
|
23
50
|
|
51
|
+
config.before_configuration do
|
52
|
+
dep = ActiveSupport::Deprecation.new("7.0", "Sidekiq")
|
53
|
+
dep.deprecate_methods(Sidekiq.singleton_class,
|
54
|
+
default_worker_options: :default_job_options,
|
55
|
+
"default_worker_options=": :default_job_options=)
|
56
|
+
end
|
57
|
+
|
24
58
|
# This hook happens after all initializers are run, just before returning
|
25
59
|
# from config/environment.rb back to sidekiq/cli.rb.
|
26
|
-
# We have to add the reloader after initialize to see if cache_classes has
|
27
|
-
# been turned on.
|
28
60
|
#
|
29
61
|
# None of this matters on the client-side, only within the Sidekiq process itself.
|
30
62
|
config.after_initialize do
|
31
|
-
Sidekiq.configure_server do |
|
32
|
-
|
33
|
-
end
|
34
|
-
end
|
35
|
-
|
36
|
-
class Reloader
|
37
|
-
def initialize(app = ::Rails.application)
|
38
|
-
@app = app
|
39
|
-
end
|
40
|
-
|
41
|
-
def call
|
42
|
-
@app.reloader.wrap do
|
43
|
-
yield
|
44
|
-
end
|
45
|
-
end
|
46
|
-
|
47
|
-
def inspect
|
48
|
-
"#<Sidekiq::Rails::Reloader @app=#{@app.class.name}>"
|
63
|
+
Sidekiq.configure_server do |config|
|
64
|
+
config[:reloader] = Sidekiq::Rails::Reloader.new
|
49
65
|
end
|
50
66
|
end
|
51
67
|
end
|
@@ -0,0 +1,154 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require "connection_pool"
|
4
|
+
require "redis_client"
|
5
|
+
require "redis_client/decorator"
|
6
|
+
require "uri"
|
7
|
+
|
8
|
+
module Sidekiq
|
9
|
+
class RedisClientAdapter
|
10
|
+
BaseError = RedisClient::Error
|
11
|
+
CommandError = RedisClient::CommandError
|
12
|
+
|
13
|
+
module CompatMethods
|
14
|
+
def info
|
15
|
+
@client.call("INFO") { |i| i.lines(chomp: true).map { |l| l.split(":", 2) }.select { |l| l.size == 2 }.to_h }
|
16
|
+
end
|
17
|
+
|
18
|
+
def evalsha(sha, keys, argv)
|
19
|
+
@client.call("EVALSHA", sha, keys.size, *keys, *argv)
|
20
|
+
end
|
21
|
+
|
22
|
+
def brpoplpush(*args)
|
23
|
+
@client.blocking_call(false, "BRPOPLPUSH", *args)
|
24
|
+
end
|
25
|
+
|
26
|
+
def brpop(*args)
|
27
|
+
@client.blocking_call(false, "BRPOP", *args)
|
28
|
+
end
|
29
|
+
|
30
|
+
def set(*args)
|
31
|
+
@client.call("SET", *args) { |r| r == "OK" }
|
32
|
+
end
|
33
|
+
ruby2_keywords :set if respond_to?(:ruby2_keywords, true)
|
34
|
+
|
35
|
+
def sismember(*args)
|
36
|
+
@client.call("SISMEMBER", *args) { |c| c > 0 }
|
37
|
+
end
|
38
|
+
|
39
|
+
def exists?(key)
|
40
|
+
@client.call("EXISTS", key) { |c| c > 0 }
|
41
|
+
end
|
42
|
+
|
43
|
+
private
|
44
|
+
|
45
|
+
def method_missing(*args, &block)
|
46
|
+
@client.call(*args, *block)
|
47
|
+
end
|
48
|
+
ruby2_keywords :method_missing if respond_to?(:ruby2_keywords, true)
|
49
|
+
|
50
|
+
def respond_to_missing?(name, include_private = false)
|
51
|
+
super # Appease the linter. We can't tell what is a valid command.
|
52
|
+
end
|
53
|
+
end
|
54
|
+
|
55
|
+
CompatClient = RedisClient::Decorator.create(CompatMethods)
|
56
|
+
|
57
|
+
class CompatClient
|
58
|
+
%i[scan sscan zscan hscan].each do |method|
|
59
|
+
alias_method :"#{method}_each", method
|
60
|
+
undef_method method
|
61
|
+
end
|
62
|
+
|
63
|
+
def disconnect!
|
64
|
+
@client.close
|
65
|
+
end
|
66
|
+
|
67
|
+
def connection
|
68
|
+
{id: @client.id}
|
69
|
+
end
|
70
|
+
|
71
|
+
def redis
|
72
|
+
self
|
73
|
+
end
|
74
|
+
|
75
|
+
def _client
|
76
|
+
@client
|
77
|
+
end
|
78
|
+
|
79
|
+
def message
|
80
|
+
yield nil, @queue.pop
|
81
|
+
end
|
82
|
+
|
83
|
+
# NB: this method does not return
|
84
|
+
def subscribe(chan)
|
85
|
+
@queue = ::Queue.new
|
86
|
+
|
87
|
+
pubsub = @client.pubsub
|
88
|
+
pubsub.call("subscribe", chan)
|
89
|
+
|
90
|
+
loop do
|
91
|
+
evt = pubsub.next_event
|
92
|
+
next if evt.nil?
|
93
|
+
next unless evt[0] == "message" && evt[1] == chan
|
94
|
+
|
95
|
+
(_, _, msg) = evt
|
96
|
+
@queue << msg
|
97
|
+
yield self
|
98
|
+
end
|
99
|
+
end
|
100
|
+
end
|
101
|
+
|
102
|
+
def initialize(options)
|
103
|
+
opts = client_opts(options)
|
104
|
+
@config = if opts.key?(:sentinels)
|
105
|
+
RedisClient.sentinel(**opts)
|
106
|
+
else
|
107
|
+
RedisClient.config(**opts)
|
108
|
+
end
|
109
|
+
end
|
110
|
+
|
111
|
+
def new_client
|
112
|
+
CompatClient.new(@config.new_client)
|
113
|
+
end
|
114
|
+
|
115
|
+
private
|
116
|
+
|
117
|
+
def client_opts(options)
|
118
|
+
opts = options.dup
|
119
|
+
|
120
|
+
if opts[:namespace]
|
121
|
+
Sidekiq.logger.error("Your Redis configuration uses the namespace '#{opts[:namespace]}' but this feature isn't supported by redis-client. " \
|
122
|
+
"Either use the redis adapter or remove the namespace.")
|
123
|
+
Kernel.exit(-127)
|
124
|
+
end
|
125
|
+
|
126
|
+
opts.delete(:size)
|
127
|
+
opts.delete(:pool_timeout)
|
128
|
+
|
129
|
+
if opts[:network_timeout]
|
130
|
+
opts[:timeout] = opts[:network_timeout]
|
131
|
+
opts.delete(:network_timeout)
|
132
|
+
end
|
133
|
+
|
134
|
+
if opts[:driver]
|
135
|
+
opts[:driver] = opts[:driver].to_sym
|
136
|
+
end
|
137
|
+
|
138
|
+
opts[:name] = opts.delete(:master_name) if opts.key?(:master_name)
|
139
|
+
opts[:role] = opts[:role].to_sym if opts.key?(:role)
|
140
|
+
opts.delete(:url) if opts.key?(:sentinels)
|
141
|
+
|
142
|
+
# Issue #3303, redis-rb will silently retry an operation.
|
143
|
+
# This can lead to duplicate jobs if Sidekiq::Client's LPUSH
|
144
|
+
# is performed twice but I believe this is much, much rarer
|
145
|
+
# than the reconnect silently fixing a problem; we keep it
|
146
|
+
# on by default.
|
147
|
+
opts[:reconnect_attempts] ||= 1
|
148
|
+
|
149
|
+
opts
|
150
|
+
end
|
151
|
+
end
|
152
|
+
end
|
153
|
+
|
154
|
+
Sidekiq::RedisConnection.adapter = Sidekiq::RedisClientAdapter
|