sidekiq 6.3.1 → 7.1.5
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/Changes.md +285 -11
- data/LICENSE.txt +9 -0
- data/README.md +47 -34
- data/bin/sidekiq +4 -9
- data/bin/sidekiqload +207 -117
- data/bin/sidekiqmon +4 -1
- data/lib/generators/sidekiq/job_generator.rb +57 -0
- data/lib/generators/sidekiq/templates/{worker.rb.erb → job.rb.erb} +2 -2
- data/lib/generators/sidekiq/templates/{worker_spec.rb.erb → job_spec.rb.erb} +1 -1
- data/lib/generators/sidekiq/templates/{worker_test.rb.erb → job_test.rb.erb} +1 -1
- data/lib/sidekiq/api.rb +333 -190
- data/lib/sidekiq/capsule.rb +127 -0
- data/lib/sidekiq/cli.rb +86 -80
- data/lib/sidekiq/client.rb +104 -95
- data/lib/sidekiq/{util.rb → component.rb} +14 -41
- data/lib/sidekiq/config.rb +282 -0
- data/lib/sidekiq/deploy.rb +62 -0
- data/lib/sidekiq/embedded.rb +61 -0
- data/lib/sidekiq/fetch.rb +23 -24
- data/lib/sidekiq/job.rb +371 -10
- data/lib/sidekiq/job_logger.rb +16 -28
- data/lib/sidekiq/job_retry.rb +99 -58
- data/lib/sidekiq/job_util.rb +107 -0
- data/lib/sidekiq/launcher.rb +103 -95
- data/lib/sidekiq/logger.rb +9 -44
- data/lib/sidekiq/manager.rb +40 -41
- data/lib/sidekiq/metrics/query.rb +153 -0
- data/lib/sidekiq/metrics/shared.rb +95 -0
- data/lib/sidekiq/metrics/tracking.rb +136 -0
- data/lib/sidekiq/middleware/chain.rb +96 -51
- data/lib/sidekiq/middleware/current_attributes.rb +59 -16
- data/lib/sidekiq/middleware/i18n.rb +6 -4
- data/lib/sidekiq/middleware/modules.rb +21 -0
- data/lib/sidekiq/monitor.rb +17 -4
- data/lib/sidekiq/paginator.rb +17 -9
- data/lib/sidekiq/processor.rb +81 -80
- data/lib/sidekiq/rails.rb +21 -14
- data/lib/sidekiq/redis_client_adapter.rb +95 -0
- data/lib/sidekiq/redis_connection.rb +14 -82
- data/lib/sidekiq/ring_buffer.rb +29 -0
- data/lib/sidekiq/scheduled.rb +76 -38
- data/lib/sidekiq/testing/inline.rb +4 -4
- data/lib/sidekiq/testing.rb +42 -69
- data/lib/sidekiq/transaction_aware_client.rb +44 -0
- data/lib/sidekiq/version.rb +2 -1
- data/lib/sidekiq/web/action.rb +3 -3
- data/lib/sidekiq/web/application.rb +95 -11
- data/lib/sidekiq/web/csrf_protection.rb +4 -4
- data/lib/sidekiq/web/helpers.rb +58 -30
- data/lib/sidekiq/web.rb +22 -17
- data/lib/sidekiq/worker_compatibility_alias.rb +13 -0
- data/lib/sidekiq.rb +85 -202
- data/sidekiq.gemspec +12 -10
- data/web/assets/javascripts/application.js +77 -26
- data/web/assets/javascripts/base-charts.js +106 -0
- data/web/assets/javascripts/chart.min.js +13 -0
- data/web/assets/javascripts/chartjs-plugin-annotation.min.js +7 -0
- data/web/assets/javascripts/dashboard-charts.js +168 -0
- data/web/assets/javascripts/dashboard.js +3 -240
- data/web/assets/javascripts/metrics.js +264 -0
- data/web/assets/stylesheets/application-dark.css +17 -17
- data/web/assets/stylesheets/application-rtl.css +2 -91
- data/web/assets/stylesheets/application.css +69 -302
- data/web/locales/ar.yml +70 -70
- data/web/locales/cs.yml +62 -62
- data/web/locales/da.yml +60 -53
- data/web/locales/de.yml +65 -65
- data/web/locales/el.yml +43 -24
- data/web/locales/en.yml +84 -69
- data/web/locales/es.yml +68 -68
- data/web/locales/fa.yml +65 -65
- data/web/locales/fr.yml +81 -67
- data/web/locales/gd.yml +99 -0
- data/web/locales/he.yml +65 -64
- data/web/locales/hi.yml +59 -59
- data/web/locales/it.yml +53 -53
- data/web/locales/ja.yml +73 -68
- data/web/locales/ko.yml +52 -52
- data/web/locales/lt.yml +66 -66
- data/web/locales/nb.yml +61 -61
- data/web/locales/nl.yml +52 -52
- data/web/locales/pl.yml +45 -45
- data/web/locales/pt-br.yml +83 -55
- data/web/locales/pt.yml +51 -51
- data/web/locales/ru.yml +67 -66
- data/web/locales/sv.yml +53 -53
- data/web/locales/ta.yml +60 -60
- data/web/locales/uk.yml +62 -61
- data/web/locales/ur.yml +64 -64
- data/web/locales/vi.yml +67 -67
- data/web/locales/zh-cn.yml +43 -16
- data/web/locales/zh-tw.yml +42 -8
- data/web/views/_footer.erb +5 -2
- data/web/views/_job_info.erb +18 -2
- data/web/views/_metrics_period_select.erb +12 -0
- data/web/views/_nav.erb +1 -1
- data/web/views/_paging.erb +2 -0
- data/web/views/_poll_link.erb +1 -1
- data/web/views/_summary.erb +1 -1
- data/web/views/busy.erb +44 -28
- data/web/views/dashboard.erb +36 -4
- data/web/views/filtering.erb +7 -0
- data/web/views/metrics.erb +82 -0
- data/web/views/metrics_for_job.erb +68 -0
- data/web/views/morgue.erb +5 -9
- data/web/views/queue.erb +15 -15
- data/web/views/queues.erb +3 -1
- data/web/views/retries.erb +5 -9
- data/web/views/scheduled.erb +12 -13
- metadata +62 -31
- data/LICENSE +0 -9
- data/lib/generators/sidekiq/worker_generator.rb +0 -57
- data/lib/sidekiq/delay.rb +0 -41
- data/lib/sidekiq/exception_handler.rb +0 -27
- data/lib/sidekiq/extensions/action_mailer.rb +0 -48
- data/lib/sidekiq/extensions/active_record.rb +0 -43
- data/lib/sidekiq/extensions/class_methods.rb +0 -43
- data/lib/sidekiq/extensions/generic_proxy.rb +0 -33
- data/lib/sidekiq/worker.rb +0 -311
@@ -7,45 +7,88 @@ module Sidekiq
|
|
7
7
|
# This can be useful for multi-tenancy, i18n locale, timezone, any implicit
|
8
8
|
# per-request attribute. See +ActiveSupport::CurrentAttributes+.
|
9
9
|
#
|
10
|
+
# For multiple current attributes, pass an array of current attributes.
|
11
|
+
#
|
10
12
|
# @example
|
11
13
|
#
|
12
14
|
# # in your initializer
|
13
15
|
# require "sidekiq/middleware/current_attributes"
|
14
|
-
# Sidekiq::CurrentAttributes.persist(Myapp::Current)
|
16
|
+
# Sidekiq::CurrentAttributes.persist("Myapp::Current")
|
17
|
+
# # or multiple current attributes
|
18
|
+
# Sidekiq::CurrentAttributes.persist(["Myapp::Current", "Myapp::OtherCurrent"])
|
15
19
|
#
|
16
20
|
module CurrentAttributes
|
17
21
|
class Save
|
18
|
-
|
19
|
-
|
22
|
+
include Sidekiq::ClientMiddleware
|
23
|
+
|
24
|
+
def initialize(cattrs)
|
25
|
+
@cattrs = cattrs
|
20
26
|
end
|
21
27
|
|
22
28
|
def call(_, job, _, _)
|
23
|
-
|
29
|
+
@cattrs.each do |(key, strklass)|
|
30
|
+
if !job.has_key?(key)
|
31
|
+
attrs = strklass.constantize.attributes
|
32
|
+
# Retries can push the job N times, we don't
|
33
|
+
# want retries to reset cattr. #5692, #5090
|
34
|
+
job[key] = attrs if attrs.any?
|
35
|
+
end
|
36
|
+
end
|
24
37
|
yield
|
25
38
|
end
|
26
39
|
end
|
27
40
|
|
28
41
|
class Load
|
29
|
-
|
30
|
-
|
42
|
+
include Sidekiq::ServerMiddleware
|
43
|
+
|
44
|
+
def initialize(cattrs)
|
45
|
+
@cattrs = cattrs
|
31
46
|
end
|
32
47
|
|
33
48
|
def call(_, job, _, &block)
|
34
|
-
|
35
|
-
|
36
|
-
|
37
|
-
|
49
|
+
cattrs_to_reset = []
|
50
|
+
|
51
|
+
@cattrs.each do |(key, strklass)|
|
52
|
+
if job.has_key?(key)
|
53
|
+
constklass = strklass.constantize
|
54
|
+
cattrs_to_reset << constklass
|
55
|
+
|
56
|
+
job[key].each do |(attribute, value)|
|
57
|
+
constklass.public_send("#{attribute}=", value)
|
58
|
+
end
|
59
|
+
end
|
38
60
|
end
|
61
|
+
|
62
|
+
yield
|
63
|
+
ensure
|
64
|
+
cattrs_to_reset.each(&:reset)
|
39
65
|
end
|
40
66
|
end
|
41
67
|
|
42
|
-
|
43
|
-
|
44
|
-
|
68
|
+
class << self
|
69
|
+
def persist(klass_or_array, config = Sidekiq.default_configuration)
|
70
|
+
cattrs = build_cattrs_hash(klass_or_array)
|
71
|
+
|
72
|
+
config.client_middleware.add Save, cattrs
|
73
|
+
config.server_middleware.add Load, cattrs
|
74
|
+
end
|
75
|
+
|
76
|
+
private
|
77
|
+
|
78
|
+
def build_cattrs_hash(klass_or_array)
|
79
|
+
if klass_or_array.is_a?(Array)
|
80
|
+
{}.tap do |hash|
|
81
|
+
klass_or_array.each_with_index do |klass, index|
|
82
|
+
hash[key_at(index)] = klass.to_s
|
83
|
+
end
|
84
|
+
end
|
85
|
+
else
|
86
|
+
{key_at(0) => klass_or_array.to_s}
|
87
|
+
end
|
45
88
|
end
|
46
|
-
|
47
|
-
|
48
|
-
|
89
|
+
|
90
|
+
def key_at(index)
|
91
|
+
(index == 0) ? "cattr" : "cattr_#{index}"
|
49
92
|
end
|
50
93
|
end
|
51
94
|
end
|
@@ -10,16 +10,18 @@ module Sidekiq::Middleware::I18n
|
|
10
10
|
# Get the current locale and store it in the message
|
11
11
|
# to be sent to Sidekiq.
|
12
12
|
class Client
|
13
|
-
|
14
|
-
|
13
|
+
include Sidekiq::ClientMiddleware
|
14
|
+
def call(_jobclass, job, _queue, _redis)
|
15
|
+
job["locale"] ||= I18n.locale
|
15
16
|
yield
|
16
17
|
end
|
17
18
|
end
|
18
19
|
|
19
20
|
# Pull the msg locale out and set the current thread to use it.
|
20
21
|
class Server
|
21
|
-
|
22
|
-
|
22
|
+
include Sidekiq::ServerMiddleware
|
23
|
+
def call(_jobclass, job, _queue, &block)
|
24
|
+
I18n.with_locale(job.fetch("locale", I18n.default_locale), &block)
|
23
25
|
end
|
24
26
|
end
|
25
27
|
end
|
@@ -0,0 +1,21 @@
|
|
1
|
+
module Sidekiq
|
2
|
+
# Server-side middleware must import this Module in order
|
3
|
+
# to get access to server resources during `call`.
|
4
|
+
module ServerMiddleware
|
5
|
+
attr_accessor :config
|
6
|
+
def redis_pool
|
7
|
+
config.redis_pool
|
8
|
+
end
|
9
|
+
|
10
|
+
def logger
|
11
|
+
config.logger
|
12
|
+
end
|
13
|
+
|
14
|
+
def redis(&block)
|
15
|
+
config.redis(&block)
|
16
|
+
end
|
17
|
+
end
|
18
|
+
|
19
|
+
# no difference for now
|
20
|
+
ClientMiddleware = ServerMiddleware
|
21
|
+
end
|
data/lib/sidekiq/monitor.rb
CHANGED
@@ -16,8 +16,6 @@ class Sidekiq::Monitor
|
|
16
16
|
return
|
17
17
|
end
|
18
18
|
send(section)
|
19
|
-
rescue => e
|
20
|
-
puts "Couldn't get status: #{e}"
|
21
19
|
end
|
22
20
|
|
23
21
|
def all
|
@@ -49,10 +47,25 @@ class Sidekiq::Monitor
|
|
49
47
|
def processes
|
50
48
|
puts "---- Processes (#{process_set.size}) ----"
|
51
49
|
process_set.each_with_index do |process, index|
|
50
|
+
# Keep compatibility with legacy versions since we don't want to break sidekiqmon during rolling upgrades or downgrades.
|
51
|
+
#
|
52
|
+
# Before:
|
53
|
+
# ["default", "critical"]
|
54
|
+
#
|
55
|
+
# After:
|
56
|
+
# {"default" => 1, "critical" => 10}
|
57
|
+
queues =
|
58
|
+
if process["weights"]
|
59
|
+
process["weights"].sort_by { |queue| queue[0] }.map { |capsule| capsule.map { |name, weight| (weight > 0) ? "#{name}: #{weight}" : name }.join(", ") }
|
60
|
+
else
|
61
|
+
process["queues"].sort
|
62
|
+
end
|
63
|
+
|
52
64
|
puts "#{process["identity"]} #{tags_for(process)}"
|
53
65
|
puts " Started: #{Time.at(process["started_at"])} (#{time_ago(process["started_at"])})"
|
54
66
|
puts " Threads: #{process["concurrency"]} (#{process["busy"]} busy)"
|
55
|
-
puts " Queues: #{split_multiline(
|
67
|
+
puts " Queues: #{split_multiline(queues, pad: 11)}"
|
68
|
+
puts " Version: #{process["version"] || "Unknown"}" if process["version"] != Sidekiq::VERSION
|
56
69
|
puts "" unless (index + 1) == process_set.size
|
57
70
|
end
|
58
71
|
end
|
@@ -101,7 +114,7 @@ class Sidekiq::Monitor
|
|
101
114
|
tags = [
|
102
115
|
process["tag"],
|
103
116
|
process["labels"],
|
104
|
-
(process["quiet"] == "true" ? "quiet" : nil)
|
117
|
+
((process["quiet"] == "true") ? "quiet" : nil)
|
105
118
|
].flatten.compact
|
106
119
|
tags.any? ? "[#{tags.join("] [")}]" : nil
|
107
120
|
end
|
data/lib/sidekiq/paginator.rb
CHANGED
@@ -3,7 +3,7 @@
|
|
3
3
|
module Sidekiq
|
4
4
|
module Paginator
|
5
5
|
def page(key, pageidx = 1, page_size = 25, opts = nil)
|
6
|
-
current_page = pageidx.to_i < 1 ? 1 : pageidx.to_i
|
6
|
+
current_page = (pageidx.to_i < 1) ? 1 : pageidx.to_i
|
7
7
|
pageidx = current_page - 1
|
8
8
|
total_size = 0
|
9
9
|
items = []
|
@@ -16,22 +16,22 @@ module Sidekiq
|
|
16
16
|
|
17
17
|
case type
|
18
18
|
when "zset"
|
19
|
-
total_size, items = conn.multi {
|
20
|
-
|
19
|
+
total_size, items = conn.multi { |transaction|
|
20
|
+
transaction.zcard(key)
|
21
21
|
if rev
|
22
|
-
|
22
|
+
transaction.zrange(key, starting, ending, "REV", withscores: true)
|
23
23
|
else
|
24
|
-
|
24
|
+
transaction.zrange(key, starting, ending, withscores: true)
|
25
25
|
end
|
26
26
|
}
|
27
27
|
[current_page, total_size, items]
|
28
28
|
when "list"
|
29
|
-
total_size, items = conn.multi {
|
30
|
-
|
29
|
+
total_size, items = conn.multi { |transaction|
|
30
|
+
transaction.llen(key)
|
31
31
|
if rev
|
32
|
-
|
32
|
+
transaction.lrange(key, -ending - 1, -starting - 1)
|
33
33
|
else
|
34
|
-
|
34
|
+
transaction.lrange(key, starting, ending)
|
35
35
|
end
|
36
36
|
}
|
37
37
|
items.reverse! if rev
|
@@ -43,5 +43,13 @@ module Sidekiq
|
|
43
43
|
end
|
44
44
|
end
|
45
45
|
end
|
46
|
+
|
47
|
+
def page_items(items, pageidx = 1, page_size = 25)
|
48
|
+
current_page = (pageidx.to_i < 1) ? 1 : pageidx.to_i
|
49
|
+
pageidx = current_page - 1
|
50
|
+
starting = pageidx * page_size
|
51
|
+
items = items.to_a
|
52
|
+
[current_page, items.size, items[starting, page_size]]
|
53
|
+
end
|
46
54
|
end
|
47
55
|
end
|
data/lib/sidekiq/processor.rb
CHANGED
@@ -1,6 +1,5 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
-
require "sidekiq/util"
|
4
3
|
require "sidekiq/fetch"
|
5
4
|
require "sidekiq/job_logger"
|
6
5
|
require "sidekiq/job_retry"
|
@@ -11,33 +10,34 @@ module Sidekiq
|
|
11
10
|
#
|
12
11
|
# 1. fetches a job from Redis
|
13
12
|
# 2. executes the job
|
14
|
-
# a. instantiate the
|
13
|
+
# a. instantiate the job class
|
15
14
|
# b. run the middleware chain
|
16
15
|
# c. call #perform
|
17
16
|
#
|
18
|
-
# A Processor can exit due to shutdown
|
19
|
-
#
|
17
|
+
# A Processor can exit due to shutdown or due to
|
18
|
+
# an error during job execution.
|
20
19
|
#
|
21
20
|
# If an error occurs in the job execution, the
|
22
21
|
# Processor calls the Manager to create a new one
|
23
22
|
# to replace itself and exits.
|
24
23
|
#
|
25
24
|
class Processor
|
26
|
-
include
|
25
|
+
include Sidekiq::Component
|
27
26
|
|
28
27
|
attr_reader :thread
|
29
28
|
attr_reader :job
|
29
|
+
attr_reader :capsule
|
30
30
|
|
31
|
-
def initialize(
|
32
|
-
@
|
31
|
+
def initialize(capsule, &block)
|
32
|
+
@config = @capsule = capsule
|
33
|
+
@callback = block
|
33
34
|
@down = false
|
34
35
|
@done = false
|
35
36
|
@job = nil
|
36
37
|
@thread = nil
|
37
|
-
@
|
38
|
-
@
|
39
|
-
@
|
40
|
-
@retrier = Sidekiq::JobRetry.new
|
38
|
+
@reloader = Sidekiq.default_configuration[:reloader]
|
39
|
+
@job_logger = (capsule.config[:job_logger] || Sidekiq::JobLogger).new(logger)
|
40
|
+
@retrier = Sidekiq::JobRetry.new(capsule)
|
41
41
|
end
|
42
42
|
|
43
43
|
def terminate(wait = false)
|
@@ -59,33 +59,37 @@ module Sidekiq
|
|
59
59
|
end
|
60
60
|
|
61
61
|
def start
|
62
|
-
@thread ||= safe_thread("processor", &method(:run))
|
62
|
+
@thread ||= safe_thread("#{config.name}/processor", &method(:run))
|
63
63
|
end
|
64
64
|
|
65
65
|
private unless $TESTING
|
66
66
|
|
67
67
|
def run
|
68
|
+
# By setting this thread-local, Sidekiq.redis will access +Sidekiq::Capsule#redis_pool+
|
69
|
+
# instead of the global pool in +Sidekiq::Config#redis_pool+.
|
70
|
+
Thread.current[:sidekiq_capsule] = @capsule
|
71
|
+
|
68
72
|
process_one until @done
|
69
|
-
@
|
73
|
+
@callback.call(self)
|
70
74
|
rescue Sidekiq::Shutdown
|
71
|
-
@
|
75
|
+
@callback.call(self)
|
72
76
|
rescue Exception => ex
|
73
|
-
@
|
77
|
+
@callback.call(self, ex)
|
74
78
|
end
|
75
79
|
|
76
|
-
def process_one
|
80
|
+
def process_one(&block)
|
77
81
|
@job = fetch
|
78
82
|
process(@job) if @job
|
79
83
|
@job = nil
|
80
84
|
end
|
81
85
|
|
82
86
|
def get_one
|
83
|
-
|
87
|
+
uow = capsule.fetcher.retrieve_work
|
84
88
|
if @down
|
85
89
|
logger.info { "Redis is online, #{::Process.clock_gettime(::Process::CLOCK_MONOTONIC) - @down} sec downtime" }
|
86
90
|
@down = nil
|
87
91
|
end
|
88
|
-
|
92
|
+
uow
|
89
93
|
rescue Sidekiq::Shutdown
|
90
94
|
rescue => ex
|
91
95
|
handle_fetch_exception(ex)
|
@@ -129,11 +133,11 @@ module Sidekiq
|
|
129
133
|
# the Reloader. It handles code loading, db connection management, etc.
|
130
134
|
# Effectively this block denotes a "unit of work" to Rails.
|
131
135
|
@reloader.call do
|
132
|
-
klass =
|
133
|
-
|
134
|
-
|
135
|
-
@retrier.local(
|
136
|
-
yield
|
136
|
+
klass = Object.const_get(job_hash["class"])
|
137
|
+
inst = klass.new
|
138
|
+
inst.jid = job_hash["jid"]
|
139
|
+
@retrier.local(inst, jobstr, queue) do
|
140
|
+
yield inst
|
137
141
|
end
|
138
142
|
end
|
139
143
|
end
|
@@ -142,9 +146,14 @@ module Sidekiq
|
|
142
146
|
end
|
143
147
|
end
|
144
148
|
|
145
|
-
|
146
|
-
|
147
|
-
|
149
|
+
IGNORE_SHUTDOWN_INTERRUPTS = {Sidekiq::Shutdown => :never}
|
150
|
+
private_constant :IGNORE_SHUTDOWN_INTERRUPTS
|
151
|
+
ALLOW_SHUTDOWN_INTERRUPTS = {Sidekiq::Shutdown => :immediate}
|
152
|
+
private_constant :ALLOW_SHUTDOWN_INTERRUPTS
|
153
|
+
|
154
|
+
def process(uow)
|
155
|
+
jobstr = uow.job
|
156
|
+
queue = uow.queue_name
|
148
157
|
|
149
158
|
# Treat malformed JSON as a special case: job goes straight to the morgue.
|
150
159
|
job_hash = nil
|
@@ -152,48 +161,53 @@ module Sidekiq
|
|
152
161
|
job_hash = Sidekiq.load_json(jobstr)
|
153
162
|
rescue => ex
|
154
163
|
handle_exception(ex, {context: "Invalid JSON for job", jobstr: jobstr})
|
155
|
-
|
156
|
-
|
157
|
-
|
164
|
+
now = Time.now.to_f
|
165
|
+
redis do |conn|
|
166
|
+
conn.multi do |xa|
|
167
|
+
xa.zadd("dead", now.to_s, jobstr)
|
168
|
+
xa.zremrangebyscore("dead", "-inf", now - @capsule.config[:dead_timeout_in_seconds])
|
169
|
+
xa.zremrangebyrank("dead", 0, - @capsule.config[:dead_max_jobs])
|
170
|
+
end
|
171
|
+
end
|
172
|
+
return uow.acknowledge
|
158
173
|
end
|
159
174
|
|
160
175
|
ack = false
|
161
|
-
|
162
|
-
|
163
|
-
|
164
|
-
|
176
|
+
Thread.handle_interrupt(IGNORE_SHUTDOWN_INTERRUPTS) do
|
177
|
+
Thread.handle_interrupt(ALLOW_SHUTDOWN_INTERRUPTS) do
|
178
|
+
dispatch(job_hash, queue, jobstr) do |inst|
|
179
|
+
config.server_middleware.invoke(inst, job_hash, queue) do
|
180
|
+
execute_job(inst, job_hash["args"])
|
181
|
+
end
|
165
182
|
end
|
183
|
+
ack = true
|
184
|
+
rescue Sidekiq::Shutdown
|
185
|
+
# Had to force kill this job because it didn't finish
|
186
|
+
# within the timeout. Don't acknowledge the work since
|
187
|
+
# we didn't properly finish it.
|
188
|
+
rescue Sidekiq::JobRetry::Handled => h
|
189
|
+
# this is the common case: job raised error and Sidekiq::JobRetry::Handled
|
190
|
+
# signals that we created a retry successfully. We can acknowlege the job.
|
191
|
+
ack = true
|
192
|
+
e = h.cause || h
|
193
|
+
handle_exception(e, {context: "Job raised exception", job: job_hash})
|
194
|
+
raise e
|
195
|
+
rescue Exception => ex
|
196
|
+
# Unexpected error! This is very bad and indicates an exception that got past
|
197
|
+
# the retry subsystem (e.g. network partition). We won't acknowledge the job
|
198
|
+
# so it can be rescued when using Sidekiq Pro.
|
199
|
+
handle_exception(ex, {context: "Internal exception!", job: job_hash, jobstr: jobstr})
|
200
|
+
raise ex
|
166
201
|
end
|
167
|
-
ack = true
|
168
|
-
rescue Sidekiq::Shutdown
|
169
|
-
# Had to force kill this job because it didn't finish
|
170
|
-
# within the timeout. Don't acknowledge the work since
|
171
|
-
# we didn't properly finish it.
|
172
|
-
rescue Sidekiq::JobRetry::Handled => h
|
173
|
-
# this is the common case: job raised error and Sidekiq::JobRetry::Handled
|
174
|
-
# signals that we created a retry successfully. We can acknowlege the job.
|
175
|
-
ack = true
|
176
|
-
e = h.cause || h
|
177
|
-
handle_exception(e, {context: "Job raised exception", job: job_hash, jobstr: jobstr})
|
178
|
-
raise e
|
179
|
-
rescue Exception => ex
|
180
|
-
# Unexpected error! This is very bad and indicates an exception that got past
|
181
|
-
# the retry subsystem (e.g. network partition). We won't acknowledge the job
|
182
|
-
# so it can be rescued when using Sidekiq Pro.
|
183
|
-
handle_exception(ex, {context: "Internal exception!", job: job_hash, jobstr: jobstr})
|
184
|
-
raise ex
|
185
202
|
ensure
|
186
203
|
if ack
|
187
|
-
|
188
|
-
Thread.handle_interrupt(Sidekiq::Shutdown => :never) do
|
189
|
-
work.acknowledge
|
190
|
-
end
|
204
|
+
uow.acknowledge
|
191
205
|
end
|
192
206
|
end
|
193
207
|
end
|
194
208
|
|
195
|
-
def execute_job(
|
196
|
-
|
209
|
+
def execute_job(inst, cloned_args)
|
210
|
+
inst.perform(*cloned_args)
|
197
211
|
end
|
198
212
|
|
199
213
|
# Ruby doesn't provide atomic counters out of the box so we'll
|
@@ -219,39 +233,39 @@ module Sidekiq
|
|
219
233
|
end
|
220
234
|
|
221
235
|
# jruby's Hash implementation is not threadsafe, so we wrap it in a mutex here
|
222
|
-
class
|
236
|
+
class SharedWorkState
|
223
237
|
def initialize
|
224
|
-
@
|
238
|
+
@work_state = {}
|
225
239
|
@lock = Mutex.new
|
226
240
|
end
|
227
241
|
|
228
242
|
def set(tid, hash)
|
229
|
-
@lock.synchronize { @
|
243
|
+
@lock.synchronize { @work_state[tid] = hash }
|
230
244
|
end
|
231
245
|
|
232
246
|
def delete(tid)
|
233
|
-
@lock.synchronize { @
|
247
|
+
@lock.synchronize { @work_state.delete(tid) }
|
234
248
|
end
|
235
249
|
|
236
250
|
def dup
|
237
|
-
@lock.synchronize { @
|
251
|
+
@lock.synchronize { @work_state.dup }
|
238
252
|
end
|
239
253
|
|
240
254
|
def size
|
241
|
-
@lock.synchronize { @
|
255
|
+
@lock.synchronize { @work_state.size }
|
242
256
|
end
|
243
257
|
|
244
258
|
def clear
|
245
|
-
@lock.synchronize { @
|
259
|
+
@lock.synchronize { @work_state.clear }
|
246
260
|
end
|
247
261
|
end
|
248
262
|
|
249
263
|
PROCESSED = Counter.new
|
250
264
|
FAILURE = Counter.new
|
251
|
-
|
265
|
+
WORK_STATE = SharedWorkState.new
|
252
266
|
|
253
267
|
def stats(jobstr, queue)
|
254
|
-
|
268
|
+
WORK_STATE.set(tid, {queue: queue, payload: jobstr, run_at: Time.now.to_i})
|
255
269
|
|
256
270
|
begin
|
257
271
|
yield
|
@@ -259,22 +273,9 @@ module Sidekiq
|
|
259
273
|
FAILURE.incr
|
260
274
|
raise
|
261
275
|
ensure
|
262
|
-
|
276
|
+
WORK_STATE.delete(tid)
|
263
277
|
PROCESSED.incr
|
264
278
|
end
|
265
279
|
end
|
266
|
-
|
267
|
-
def constantize(str)
|
268
|
-
return Object.const_get(str) unless str.include?("::")
|
269
|
-
|
270
|
-
names = str.split("::")
|
271
|
-
names.shift if names.empty? || names.first.empty?
|
272
|
-
|
273
|
-
names.inject(Object) do |constant, name|
|
274
|
-
# the false flag limits search for name to under the constant namespace
|
275
|
-
# which mimics Rails' behaviour
|
276
|
-
constant.const_get(name, false)
|
277
|
-
end
|
278
|
-
end
|
279
280
|
end
|
280
281
|
end
|
data/lib/sidekiq/rails.rb
CHANGED
@@ -1,6 +1,7 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
-
require "sidekiq/
|
3
|
+
require "sidekiq/job"
|
4
|
+
require "rails"
|
4
5
|
|
5
6
|
module Sidekiq
|
6
7
|
class Rails < ::Rails::Engine
|
@@ -10,7 +11,8 @@ module Sidekiq
|
|
10
11
|
end
|
11
12
|
|
12
13
|
def call
|
13
|
-
|
14
|
+
params = (::Rails::VERSION::STRING >= "7.1") ? {source: "job.sidekiq"} : {}
|
15
|
+
@app.reloader.wrap(**params) do
|
14
16
|
yield
|
15
17
|
end
|
16
18
|
end
|
@@ -22,7 +24,7 @@ module Sidekiq
|
|
22
24
|
|
23
25
|
# By including the Options module, we allow AJs to directly control sidekiq features
|
24
26
|
# via the *sidekiq_options* class method and, for instance, not use AJ's retry system.
|
25
|
-
# AJ retries don't show up in the Sidekiq UI Retries tab, save any error data, can't be
|
27
|
+
# AJ retries don't show up in the Sidekiq UI Retries tab, don't save any error data, can't be
|
26
28
|
# manually retried, don't automatically die, etc.
|
27
29
|
#
|
28
30
|
# class SomeJob < ActiveJob::Base
|
@@ -33,18 +35,13 @@ module Sidekiq
|
|
33
35
|
# end
|
34
36
|
initializer "sidekiq.active_job_integration" do
|
35
37
|
ActiveSupport.on_load(:active_job) do
|
36
|
-
include ::Sidekiq::
|
38
|
+
include ::Sidekiq::Job::Options unless respond_to?(:sidekiq_options)
|
37
39
|
end
|
38
40
|
end
|
39
41
|
|
40
|
-
initializer "sidekiq.
|
41
|
-
Sidekiq.configure_server do |
|
42
|
-
|
43
|
-
# it will appear in the Sidekiq console with all of the job context. See #5021 and
|
44
|
-
# https://github.com/rails/rails/blob/b5f2b550f69a99336482739000c58e4e04e033aa/railties/lib/rails/commands/server/server_command.rb#L82-L84
|
45
|
-
unless ::ActiveSupport::Logger.logger_outputs_to?(::Rails.logger, $stdout)
|
46
|
-
::Rails.logger.extend(::ActiveSupport::Logger.broadcast(::Sidekiq.logger))
|
47
|
-
end
|
42
|
+
initializer "sidekiq.backtrace_cleaner" do
|
43
|
+
Sidekiq.configure_server do |config|
|
44
|
+
config[:backtrace_cleaner] = ->(backtrace) { ::Rails.backtrace_cleaner.clean(backtrace) }
|
48
45
|
end
|
49
46
|
end
|
50
47
|
|
@@ -53,8 +50,18 @@ module Sidekiq
|
|
53
50
|
#
|
54
51
|
# None of this matters on the client-side, only within the Sidekiq process itself.
|
55
52
|
config.after_initialize do
|
56
|
-
Sidekiq.configure_server do |
|
57
|
-
|
53
|
+
Sidekiq.configure_server do |config|
|
54
|
+
config[:reloader] = Sidekiq::Rails::Reloader.new
|
55
|
+
|
56
|
+
# This is the integration code necessary so that if a job uses `Rails.logger.info "Hello"`,
|
57
|
+
# it will appear in the Sidekiq console with all of the job context.
|
58
|
+
unless ::Rails.logger == config.logger || ::ActiveSupport::Logger.logger_outputs_to?(::Rails.logger, $stdout)
|
59
|
+
if ::Rails::VERSION::STRING < "7.1"
|
60
|
+
::Rails.logger.extend(::ActiveSupport::Logger.broadcast(config.logger))
|
61
|
+
else
|
62
|
+
::Rails.logger.broadcast_to(config.logger)
|
63
|
+
end
|
64
|
+
end
|
58
65
|
end
|
59
66
|
end
|
60
67
|
end
|