sidekiq 6.4.0 → 6.5.12
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/Changes.md +119 -1
- data/README.md +6 -1
- data/bin/sidekiq +3 -3
- data/bin/sidekiqload +70 -66
- data/bin/sidekiqmon +1 -1
- data/lib/sidekiq/api.rb +255 -100
- data/lib/sidekiq/cli.rb +60 -38
- data/lib/sidekiq/client.rb +44 -30
- data/lib/sidekiq/component.rb +65 -0
- data/lib/sidekiq/delay.rb +2 -2
- data/lib/sidekiq/extensions/action_mailer.rb +2 -2
- data/lib/sidekiq/extensions/active_record.rb +2 -2
- data/lib/sidekiq/extensions/class_methods.rb +2 -2
- data/lib/sidekiq/extensions/generic_proxy.rb +3 -3
- data/lib/sidekiq/fetch.rb +20 -18
- data/lib/sidekiq/job_logger.rb +15 -27
- data/lib/sidekiq/job_retry.rb +73 -52
- data/lib/sidekiq/job_util.rb +15 -9
- data/lib/sidekiq/launcher.rb +58 -54
- data/lib/sidekiq/logger.rb +8 -18
- data/lib/sidekiq/manager.rb +28 -25
- data/lib/sidekiq/metrics/deploy.rb +47 -0
- data/lib/sidekiq/metrics/query.rb +153 -0
- data/lib/sidekiq/metrics/shared.rb +94 -0
- data/lib/sidekiq/metrics/tracking.rb +134 -0
- data/lib/sidekiq/middleware/chain.rb +82 -38
- data/lib/sidekiq/middleware/current_attributes.rb +18 -12
- data/lib/sidekiq/middleware/i18n.rb +6 -4
- data/lib/sidekiq/middleware/modules.rb +21 -0
- data/lib/sidekiq/monitor.rb +2 -2
- data/lib/sidekiq/paginator.rb +17 -9
- data/lib/sidekiq/processor.rb +47 -41
- data/lib/sidekiq/rails.rb +19 -13
- data/lib/sidekiq/redis_client_adapter.rb +154 -0
- data/lib/sidekiq/redis_connection.rb +80 -49
- data/lib/sidekiq/ring_buffer.rb +29 -0
- data/lib/sidekiq/scheduled.rb +53 -24
- data/lib/sidekiq/testing/inline.rb +4 -4
- data/lib/sidekiq/testing.rb +37 -36
- data/lib/sidekiq/transaction_aware_client.rb +45 -0
- data/lib/sidekiq/version.rb +1 -1
- data/lib/sidekiq/web/action.rb +3 -3
- data/lib/sidekiq/web/application.rb +21 -5
- data/lib/sidekiq/web/csrf_protection.rb +2 -2
- data/lib/sidekiq/web/helpers.rb +21 -8
- data/lib/sidekiq/web.rb +8 -4
- data/lib/sidekiq/worker.rb +26 -20
- data/lib/sidekiq.rb +107 -31
- data/sidekiq.gemspec +2 -2
- data/web/assets/javascripts/application.js +59 -26
- data/web/assets/javascripts/chart.min.js +13 -0
- data/web/assets/javascripts/chartjs-plugin-annotation.min.js +7 -0
- data/web/assets/javascripts/dashboard.js +0 -17
- data/web/assets/javascripts/graph.js +16 -0
- data/web/assets/javascripts/metrics.js +262 -0
- data/web/assets/stylesheets/application.css +45 -1
- data/web/locales/el.yml +43 -19
- data/web/locales/en.yml +7 -0
- data/web/locales/ja.yml +7 -0
- data/web/locales/pt-br.yml +27 -9
- data/web/locales/zh-cn.yml +36 -11
- data/web/locales/zh-tw.yml +32 -7
- data/web/views/_nav.erb +1 -1
- data/web/views/_summary.erb +1 -1
- data/web/views/busy.erb +9 -4
- data/web/views/dashboard.erb +1 -0
- data/web/views/metrics.erb +69 -0
- data/web/views/metrics_for_job.erb +87 -0
- data/web/views/queue.erb +5 -1
- metadata +34 -9
- data/lib/sidekiq/exception_handler.rb +0 -27
- data/lib/sidekiq/util.rb +0 -108
@@ -1,82 +1,102 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
+
require "sidekiq/middleware/modules"
|
4
|
+
|
3
5
|
module Sidekiq
|
4
6
|
# Middleware is code configured to run before/after
|
5
|
-
# a
|
7
|
+
# a job is processed. It is patterned after Rack
|
6
8
|
# middleware. Middleware exists for the client side
|
7
9
|
# (pushing jobs onto the queue) as well as the server
|
8
10
|
# side (when jobs are actually processed).
|
9
11
|
#
|
12
|
+
# Callers will register middleware Classes and Sidekiq will
|
13
|
+
# create new instances of the middleware for every job. This
|
14
|
+
# is important so that instance state is not shared accidentally
|
15
|
+
# between job executions.
|
16
|
+
#
|
10
17
|
# To add middleware for the client:
|
11
18
|
#
|
12
|
-
#
|
13
|
-
#
|
14
|
-
#
|
19
|
+
# Sidekiq.configure_client do |config|
|
20
|
+
# config.client_middleware do |chain|
|
21
|
+
# chain.add MyClientHook
|
22
|
+
# end
|
15
23
|
# end
|
16
|
-
# end
|
17
24
|
#
|
18
25
|
# To modify middleware for the server, just call
|
19
26
|
# with another block:
|
20
27
|
#
|
21
|
-
#
|
22
|
-
#
|
23
|
-
#
|
24
|
-
#
|
28
|
+
# Sidekiq.configure_server do |config|
|
29
|
+
# config.server_middleware do |chain|
|
30
|
+
# chain.add MyServerHook
|
31
|
+
# chain.remove ActiveRecord
|
32
|
+
# end
|
25
33
|
# end
|
26
|
-
# end
|
27
34
|
#
|
28
35
|
# To insert immediately preceding another entry:
|
29
36
|
#
|
30
|
-
#
|
31
|
-
#
|
32
|
-
#
|
37
|
+
# Sidekiq.configure_client do |config|
|
38
|
+
# config.client_middleware do |chain|
|
39
|
+
# chain.insert_before ActiveRecord, MyClientHook
|
40
|
+
# end
|
33
41
|
# end
|
34
|
-
# end
|
35
42
|
#
|
36
43
|
# To insert immediately after another entry:
|
37
44
|
#
|
38
|
-
#
|
39
|
-
#
|
40
|
-
#
|
45
|
+
# Sidekiq.configure_client do |config|
|
46
|
+
# config.client_middleware do |chain|
|
47
|
+
# chain.insert_after ActiveRecord, MyClientHook
|
48
|
+
# end
|
41
49
|
# end
|
42
|
-
# end
|
43
50
|
#
|
44
51
|
# This is an example of a minimal server middleware:
|
45
52
|
#
|
46
|
-
#
|
47
|
-
#
|
48
|
-
#
|
49
|
-
#
|
50
|
-
#
|
53
|
+
# class MyServerHook
|
54
|
+
# include Sidekiq::ServerMiddleware
|
55
|
+
#
|
56
|
+
# def call(job_instance, msg, queue)
|
57
|
+
# logger.info "Before job"
|
58
|
+
# redis {|conn| conn.get("foo") } # do something in Redis
|
59
|
+
# yield
|
60
|
+
# logger.info "After job"
|
61
|
+
# end
|
51
62
|
# end
|
52
|
-
# end
|
53
63
|
#
|
54
64
|
# This is an example of a minimal client middleware, note
|
55
65
|
# the method must return the result or the job will not push
|
56
66
|
# to Redis:
|
57
67
|
#
|
58
|
-
#
|
59
|
-
#
|
60
|
-
#
|
61
|
-
#
|
62
|
-
#
|
63
|
-
#
|
68
|
+
# class MyClientHook
|
69
|
+
# include Sidekiq::ClientMiddleware
|
70
|
+
#
|
71
|
+
# def call(job_class, msg, queue, redis_pool)
|
72
|
+
# logger.info "Before push"
|
73
|
+
# result = yield
|
74
|
+
# logger.info "After push"
|
75
|
+
# result
|
76
|
+
# end
|
64
77
|
# end
|
65
|
-
# end
|
66
78
|
#
|
67
79
|
module Middleware
|
68
80
|
class Chain
|
69
81
|
include Enumerable
|
70
82
|
|
83
|
+
# A unique instance of the middleware chain is created for
|
84
|
+
# each job executed in order to be thread-safe.
|
85
|
+
# @param copy [Sidekiq::Middleware::Chain] New instance of Chain
|
86
|
+
# @returns nil
|
71
87
|
def initialize_copy(copy)
|
72
88
|
copy.instance_variable_set(:@entries, entries.dup)
|
89
|
+
nil
|
73
90
|
end
|
74
91
|
|
92
|
+
# Iterate through each middleware in the chain
|
75
93
|
def each(&block)
|
76
94
|
entries.each(&block)
|
77
95
|
end
|
78
96
|
|
79
|
-
|
97
|
+
# @api private
|
98
|
+
def initialize(config = nil) # :nodoc:
|
99
|
+
@config = config
|
80
100
|
@entries = nil
|
81
101
|
yield self if block_given?
|
82
102
|
end
|
@@ -85,38 +105,55 @@ module Sidekiq
|
|
85
105
|
@entries ||= []
|
86
106
|
end
|
87
107
|
|
108
|
+
# Remove all middleware matching the given Class
|
109
|
+
# @param klass [Class]
|
88
110
|
def remove(klass)
|
89
111
|
entries.delete_if { |entry| entry.klass == klass }
|
90
112
|
end
|
91
113
|
|
114
|
+
# Add the given middleware to the end of the chain.
|
115
|
+
# Sidekiq will call `klass.new(*args)` to create a clean
|
116
|
+
# copy of your middleware for every job executed.
|
117
|
+
#
|
118
|
+
# chain.add(Statsd::Metrics, { collector: "localhost:8125" })
|
119
|
+
#
|
120
|
+
# @param klass [Class] Your middleware class
|
121
|
+
# @param *args [Array<Object>] Set of arguments to pass to every instance of your middleware
|
92
122
|
def add(klass, *args)
|
93
123
|
remove(klass)
|
94
|
-
entries << Entry.new(klass, *args)
|
124
|
+
entries << Entry.new(@config, klass, *args)
|
95
125
|
end
|
96
126
|
|
127
|
+
# Identical to {#add} except the middleware is added to the front of the chain.
|
97
128
|
def prepend(klass, *args)
|
98
129
|
remove(klass)
|
99
|
-
entries.insert(0, Entry.new(klass, *args))
|
130
|
+
entries.insert(0, Entry.new(@config, klass, *args))
|
100
131
|
end
|
101
132
|
|
133
|
+
# Inserts +newklass+ before +oldklass+ in the chain.
|
134
|
+
# Useful if one middleware must run before another middleware.
|
102
135
|
def insert_before(oldklass, newklass, *args)
|
103
136
|
i = entries.index { |entry| entry.klass == newklass }
|
104
|
-
new_entry = i.nil? ? Entry.new(newklass, *args) : entries.delete_at(i)
|
137
|
+
new_entry = i.nil? ? Entry.new(@config, newklass, *args) : entries.delete_at(i)
|
105
138
|
i = entries.index { |entry| entry.klass == oldklass } || 0
|
106
139
|
entries.insert(i, new_entry)
|
107
140
|
end
|
108
141
|
|
142
|
+
# Inserts +newklass+ after +oldklass+ in the chain.
|
143
|
+
# Useful if one middleware must run after another middleware.
|
109
144
|
def insert_after(oldklass, newklass, *args)
|
110
145
|
i = entries.index { |entry| entry.klass == newklass }
|
111
|
-
new_entry = i.nil? ? Entry.new(newklass, *args) : entries.delete_at(i)
|
146
|
+
new_entry = i.nil? ? Entry.new(@config, newklass, *args) : entries.delete_at(i)
|
112
147
|
i = entries.index { |entry| entry.klass == oldklass } || entries.count - 1
|
113
148
|
entries.insert(i + 1, new_entry)
|
114
149
|
end
|
115
150
|
|
151
|
+
# @return [Boolean] if the given class is already in the chain
|
116
152
|
def exists?(klass)
|
117
153
|
any? { |entry| entry.klass == klass }
|
118
154
|
end
|
119
155
|
|
156
|
+
# @return [Boolean] if the chain contains no middleware
|
120
157
|
def empty?
|
121
158
|
@entries.nil? || @entries.empty?
|
122
159
|
end
|
@@ -129,6 +166,8 @@ module Sidekiq
|
|
129
166
|
entries.clear
|
130
167
|
end
|
131
168
|
|
169
|
+
# Used by Sidekiq to execute the middleware at runtime
|
170
|
+
# @api private
|
132
171
|
def invoke(*args)
|
133
172
|
return yield if empty?
|
134
173
|
|
@@ -146,16 +185,21 @@ module Sidekiq
|
|
146
185
|
|
147
186
|
private
|
148
187
|
|
188
|
+
# Represents each link in the middleware chain
|
189
|
+
# @api private
|
149
190
|
class Entry
|
150
191
|
attr_reader :klass
|
151
192
|
|
152
|
-
def initialize(klass, *args)
|
193
|
+
def initialize(config, klass, *args)
|
194
|
+
@config = config
|
153
195
|
@klass = klass
|
154
196
|
@args = args
|
155
197
|
end
|
156
198
|
|
157
199
|
def make_new
|
158
|
-
@klass.new(*@args)
|
200
|
+
x = @klass.new(*@args)
|
201
|
+
x.config = @config if @config && x.respond_to?(:config=)
|
202
|
+
x
|
159
203
|
end
|
160
204
|
end
|
161
205
|
end
|
@@ -11,33 +11,39 @@ module Sidekiq
|
|
11
11
|
#
|
12
12
|
# # in your initializer
|
13
13
|
# require "sidekiq/middleware/current_attributes"
|
14
|
-
# Sidekiq::CurrentAttributes.persist(Myapp::Current)
|
14
|
+
# Sidekiq::CurrentAttributes.persist("Myapp::Current")
|
15
15
|
#
|
16
16
|
module CurrentAttributes
|
17
17
|
class Save
|
18
|
+
include Sidekiq::ClientMiddleware
|
19
|
+
|
18
20
|
def initialize(cattr)
|
19
|
-
@
|
21
|
+
@strklass = cattr
|
20
22
|
end
|
21
23
|
|
22
24
|
def call(_, job, _, _)
|
23
|
-
attrs = @
|
24
|
-
if
|
25
|
-
job
|
26
|
-
|
27
|
-
|
25
|
+
attrs = @strklass.constantize.attributes
|
26
|
+
if attrs.any?
|
27
|
+
if job.has_key?("cattr")
|
28
|
+
job["cattr"].merge!(attrs)
|
29
|
+
else
|
30
|
+
job["cattr"] = attrs
|
31
|
+
end
|
28
32
|
end
|
29
33
|
yield
|
30
34
|
end
|
31
35
|
end
|
32
36
|
|
33
37
|
class Load
|
38
|
+
include Sidekiq::ServerMiddleware
|
39
|
+
|
34
40
|
def initialize(cattr)
|
35
|
-
@
|
41
|
+
@strklass = cattr
|
36
42
|
end
|
37
43
|
|
38
44
|
def call(_, job, _, &block)
|
39
45
|
if job.has_key?("cattr")
|
40
|
-
@
|
46
|
+
@strklass.constantize.set(job["cattr"], &block)
|
41
47
|
else
|
42
48
|
yield
|
43
49
|
end
|
@@ -46,11 +52,11 @@ module Sidekiq
|
|
46
52
|
|
47
53
|
def self.persist(klass)
|
48
54
|
Sidekiq.configure_client do |config|
|
49
|
-
config.client_middleware.add Save, klass
|
55
|
+
config.client_middleware.add Save, klass.to_s
|
50
56
|
end
|
51
57
|
Sidekiq.configure_server do |config|
|
52
|
-
config.client_middleware.add Save, klass
|
53
|
-
config.server_middleware.add Load, klass
|
58
|
+
config.client_middleware.add Save, klass.to_s
|
59
|
+
config.server_middleware.add Load, klass.to_s
|
54
60
|
end
|
55
61
|
end
|
56
62
|
end
|
@@ -10,16 +10,18 @@ module Sidekiq::Middleware::I18n
|
|
10
10
|
# Get the current locale and store it in the message
|
11
11
|
# to be sent to Sidekiq.
|
12
12
|
class Client
|
13
|
-
|
14
|
-
|
13
|
+
include Sidekiq::ClientMiddleware
|
14
|
+
def call(_jobclass, job, _queue, _redis)
|
15
|
+
job["locale"] ||= I18n.locale
|
15
16
|
yield
|
16
17
|
end
|
17
18
|
end
|
18
19
|
|
19
20
|
# Pull the msg locale out and set the current thread to use it.
|
20
21
|
class Server
|
21
|
-
|
22
|
-
|
22
|
+
include Sidekiq::ServerMiddleware
|
23
|
+
def call(_jobclass, job, _queue, &block)
|
24
|
+
I18n.with_locale(job.fetch("locale", I18n.default_locale), &block)
|
23
25
|
end
|
24
26
|
end
|
25
27
|
end
|
@@ -0,0 +1,21 @@
|
|
1
|
+
module Sidekiq
|
2
|
+
# Server-side middleware must import this Module in order
|
3
|
+
# to get access to server resources during `call`.
|
4
|
+
module ServerMiddleware
|
5
|
+
attr_accessor :config
|
6
|
+
def redis_pool
|
7
|
+
config.redis_pool
|
8
|
+
end
|
9
|
+
|
10
|
+
def logger
|
11
|
+
config.logger
|
12
|
+
end
|
13
|
+
|
14
|
+
def redis(&block)
|
15
|
+
config.redis(&block)
|
16
|
+
end
|
17
|
+
end
|
18
|
+
|
19
|
+
# no difference for now
|
20
|
+
ClientMiddleware = ServerMiddleware
|
21
|
+
end
|
data/lib/sidekiq/monitor.rb
CHANGED
@@ -17,7 +17,7 @@ class Sidekiq::Monitor
|
|
17
17
|
end
|
18
18
|
send(section)
|
19
19
|
rescue => e
|
20
|
-
|
20
|
+
abort "Couldn't get status: #{e}"
|
21
21
|
end
|
22
22
|
|
23
23
|
def all
|
@@ -101,7 +101,7 @@ class Sidekiq::Monitor
|
|
101
101
|
tags = [
|
102
102
|
process["tag"],
|
103
103
|
process["labels"],
|
104
|
-
(process["quiet"] == "true" ? "quiet" : nil)
|
104
|
+
((process["quiet"] == "true") ? "quiet" : nil)
|
105
105
|
].flatten.compact
|
106
106
|
tags.any? ? "[#{tags.join("] [")}]" : nil
|
107
107
|
end
|
data/lib/sidekiq/paginator.rb
CHANGED
@@ -3,7 +3,7 @@
|
|
3
3
|
module Sidekiq
|
4
4
|
module Paginator
|
5
5
|
def page(key, pageidx = 1, page_size = 25, opts = nil)
|
6
|
-
current_page = pageidx.to_i < 1 ? 1 : pageidx.to_i
|
6
|
+
current_page = (pageidx.to_i < 1) ? 1 : pageidx.to_i
|
7
7
|
pageidx = current_page - 1
|
8
8
|
total_size = 0
|
9
9
|
items = []
|
@@ -16,22 +16,22 @@ module Sidekiq
|
|
16
16
|
|
17
17
|
case type
|
18
18
|
when "zset"
|
19
|
-
total_size, items = conn.multi {
|
20
|
-
|
19
|
+
total_size, items = conn.multi { |transaction|
|
20
|
+
transaction.zcard(key)
|
21
21
|
if rev
|
22
|
-
|
22
|
+
transaction.zrevrange(key, starting, ending, withscores: true)
|
23
23
|
else
|
24
|
-
|
24
|
+
transaction.zrange(key, starting, ending, withscores: true)
|
25
25
|
end
|
26
26
|
}
|
27
27
|
[current_page, total_size, items]
|
28
28
|
when "list"
|
29
|
-
total_size, items = conn.multi {
|
30
|
-
|
29
|
+
total_size, items = conn.multi { |transaction|
|
30
|
+
transaction.llen(key)
|
31
31
|
if rev
|
32
|
-
|
32
|
+
transaction.lrange(key, -ending - 1, -starting - 1)
|
33
33
|
else
|
34
|
-
|
34
|
+
transaction.lrange(key, starting, ending)
|
35
35
|
end
|
36
36
|
}
|
37
37
|
items.reverse! if rev
|
@@ -43,5 +43,13 @@ module Sidekiq
|
|
43
43
|
end
|
44
44
|
end
|
45
45
|
end
|
46
|
+
|
47
|
+
def page_items(items, pageidx = 1, page_size = 25)
|
48
|
+
current_page = (pageidx.to_i < 1) ? 1 : pageidx.to_i
|
49
|
+
pageidx = current_page - 1
|
50
|
+
starting = pageidx * page_size
|
51
|
+
items = items.to_a
|
52
|
+
[current_page, items.size, items[starting, page_size]]
|
53
|
+
end
|
46
54
|
end
|
47
55
|
end
|
data/lib/sidekiq/processor.rb
CHANGED
@@ -1,6 +1,5 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
-
require "sidekiq/util"
|
4
3
|
require "sidekiq/fetch"
|
5
4
|
require "sidekiq/job_logger"
|
6
5
|
require "sidekiq/job_retry"
|
@@ -11,33 +10,34 @@ module Sidekiq
|
|
11
10
|
#
|
12
11
|
# 1. fetches a job from Redis
|
13
12
|
# 2. executes the job
|
14
|
-
# a. instantiate the
|
13
|
+
# a. instantiate the job class
|
15
14
|
# b. run the middleware chain
|
16
15
|
# c. call #perform
|
17
16
|
#
|
18
|
-
# A Processor can exit due to shutdown
|
19
|
-
#
|
17
|
+
# A Processor can exit due to shutdown or due to
|
18
|
+
# an error during job execution.
|
20
19
|
#
|
21
20
|
# If an error occurs in the job execution, the
|
22
21
|
# Processor calls the Manager to create a new one
|
23
22
|
# to replace itself and exits.
|
24
23
|
#
|
25
24
|
class Processor
|
26
|
-
include
|
25
|
+
include Sidekiq::Component
|
27
26
|
|
28
27
|
attr_reader :thread
|
29
28
|
attr_reader :job
|
30
29
|
|
31
|
-
def initialize(
|
32
|
-
@
|
30
|
+
def initialize(options, &block)
|
31
|
+
@callback = block
|
33
32
|
@down = false
|
34
33
|
@done = false
|
35
34
|
@job = nil
|
36
35
|
@thread = nil
|
36
|
+
@config = options
|
37
37
|
@strategy = options[:fetch]
|
38
38
|
@reloader = options[:reloader] || proc { |&block| block.call }
|
39
39
|
@job_logger = (options[:job_logger] || Sidekiq::JobLogger).new
|
40
|
-
@retrier = Sidekiq::JobRetry.new
|
40
|
+
@retrier = Sidekiq::JobRetry.new(options)
|
41
41
|
end
|
42
42
|
|
43
43
|
def terminate(wait = false)
|
@@ -66,26 +66,26 @@ module Sidekiq
|
|
66
66
|
|
67
67
|
def run
|
68
68
|
process_one until @done
|
69
|
-
@
|
69
|
+
@callback.call(self)
|
70
70
|
rescue Sidekiq::Shutdown
|
71
|
-
@
|
71
|
+
@callback.call(self)
|
72
72
|
rescue Exception => ex
|
73
|
-
@
|
73
|
+
@callback.call(self, ex)
|
74
74
|
end
|
75
75
|
|
76
|
-
def process_one
|
76
|
+
def process_one(&block)
|
77
77
|
@job = fetch
|
78
78
|
process(@job) if @job
|
79
79
|
@job = nil
|
80
80
|
end
|
81
81
|
|
82
82
|
def get_one
|
83
|
-
|
83
|
+
uow = @strategy.retrieve_work
|
84
84
|
if @down
|
85
85
|
logger.info { "Redis is online, #{::Process.clock_gettime(::Process::CLOCK_MONOTONIC) - @down} sec downtime" }
|
86
86
|
@down = nil
|
87
87
|
end
|
88
|
-
|
88
|
+
uow
|
89
89
|
rescue Sidekiq::Shutdown
|
90
90
|
rescue => ex
|
91
91
|
handle_fetch_exception(ex)
|
@@ -130,10 +130,10 @@ module Sidekiq
|
|
130
130
|
# Effectively this block denotes a "unit of work" to Rails.
|
131
131
|
@reloader.call do
|
132
132
|
klass = constantize(job_hash["class"])
|
133
|
-
|
134
|
-
|
135
|
-
@retrier.local(
|
136
|
-
yield
|
133
|
+
inst = klass.new
|
134
|
+
inst.jid = job_hash["jid"]
|
135
|
+
@retrier.local(inst, jobstr, queue) do
|
136
|
+
yield inst
|
137
137
|
end
|
138
138
|
end
|
139
139
|
end
|
@@ -142,9 +142,9 @@ module Sidekiq
|
|
142
142
|
end
|
143
143
|
end
|
144
144
|
|
145
|
-
def process(
|
146
|
-
jobstr =
|
147
|
-
queue =
|
145
|
+
def process(uow)
|
146
|
+
jobstr = uow.job
|
147
|
+
queue = uow.queue_name
|
148
148
|
|
149
149
|
# Treat malformed JSON as a special case: job goes straight to the morgue.
|
150
150
|
job_hash = nil
|
@@ -152,16 +152,22 @@ module Sidekiq
|
|
152
152
|
job_hash = Sidekiq.load_json(jobstr)
|
153
153
|
rescue => ex
|
154
154
|
handle_exception(ex, {context: "Invalid JSON for job", jobstr: jobstr})
|
155
|
-
|
156
|
-
|
157
|
-
|
155
|
+
now = Time.now.to_f
|
156
|
+
config.redis do |conn|
|
157
|
+
conn.multi do |xa|
|
158
|
+
xa.zadd("dead", now.to_s, jobstr)
|
159
|
+
xa.zremrangebyscore("dead", "-inf", now - config[:dead_timeout_in_seconds])
|
160
|
+
xa.zremrangebyrank("dead", 0, - config[:dead_max_jobs])
|
161
|
+
end
|
162
|
+
end
|
163
|
+
return uow.acknowledge
|
158
164
|
end
|
159
165
|
|
160
166
|
ack = false
|
161
167
|
begin
|
162
|
-
dispatch(job_hash, queue, jobstr) do |
|
163
|
-
|
164
|
-
execute_job(
|
168
|
+
dispatch(job_hash, queue, jobstr) do |inst|
|
169
|
+
@config.server_middleware.invoke(inst, job_hash, queue) do
|
170
|
+
execute_job(inst, job_hash["args"])
|
165
171
|
end
|
166
172
|
end
|
167
173
|
ack = true
|
@@ -174,7 +180,7 @@ module Sidekiq
|
|
174
180
|
# signals that we created a retry successfully. We can acknowlege the job.
|
175
181
|
ack = true
|
176
182
|
e = h.cause || h
|
177
|
-
handle_exception(e, {context: "Job raised exception", job: job_hash
|
183
|
+
handle_exception(e, {context: "Job raised exception", job: job_hash})
|
178
184
|
raise e
|
179
185
|
rescue Exception => ex
|
180
186
|
# Unexpected error! This is very bad and indicates an exception that got past
|
@@ -186,14 +192,14 @@ module Sidekiq
|
|
186
192
|
if ack
|
187
193
|
# We don't want a shutdown signal to interrupt job acknowledgment.
|
188
194
|
Thread.handle_interrupt(Sidekiq::Shutdown => :never) do
|
189
|
-
|
195
|
+
uow.acknowledge
|
190
196
|
end
|
191
197
|
end
|
192
198
|
end
|
193
199
|
end
|
194
200
|
|
195
|
-
def execute_job(
|
196
|
-
|
201
|
+
def execute_job(inst, cloned_args)
|
202
|
+
inst.perform(*cloned_args)
|
197
203
|
end
|
198
204
|
|
199
205
|
# Ruby doesn't provide atomic counters out of the box so we'll
|
@@ -219,39 +225,39 @@ module Sidekiq
|
|
219
225
|
end
|
220
226
|
|
221
227
|
# jruby's Hash implementation is not threadsafe, so we wrap it in a mutex here
|
222
|
-
class
|
228
|
+
class SharedWorkState
|
223
229
|
def initialize
|
224
|
-
@
|
230
|
+
@work_state = {}
|
225
231
|
@lock = Mutex.new
|
226
232
|
end
|
227
233
|
|
228
234
|
def set(tid, hash)
|
229
|
-
@lock.synchronize { @
|
235
|
+
@lock.synchronize { @work_state[tid] = hash }
|
230
236
|
end
|
231
237
|
|
232
238
|
def delete(tid)
|
233
|
-
@lock.synchronize { @
|
239
|
+
@lock.synchronize { @work_state.delete(tid) }
|
234
240
|
end
|
235
241
|
|
236
242
|
def dup
|
237
|
-
@lock.synchronize { @
|
243
|
+
@lock.synchronize { @work_state.dup }
|
238
244
|
end
|
239
245
|
|
240
246
|
def size
|
241
|
-
@lock.synchronize { @
|
247
|
+
@lock.synchronize { @work_state.size }
|
242
248
|
end
|
243
249
|
|
244
250
|
def clear
|
245
|
-
@lock.synchronize { @
|
251
|
+
@lock.synchronize { @work_state.clear }
|
246
252
|
end
|
247
253
|
end
|
248
254
|
|
249
255
|
PROCESSED = Counter.new
|
250
256
|
FAILURE = Counter.new
|
251
|
-
|
257
|
+
WORK_STATE = SharedWorkState.new
|
252
258
|
|
253
259
|
def stats(jobstr, queue)
|
254
|
-
|
260
|
+
WORK_STATE.set(tid, {queue: queue, payload: jobstr, run_at: Time.now.to_i})
|
255
261
|
|
256
262
|
begin
|
257
263
|
yield
|
@@ -259,7 +265,7 @@ module Sidekiq
|
|
259
265
|
FAILURE.incr
|
260
266
|
raise
|
261
267
|
ensure
|
262
|
-
|
268
|
+
WORK_STATE.delete(tid)
|
263
269
|
PROCESSED.incr
|
264
270
|
end
|
265
271
|
end
|
data/lib/sidekiq/rails.rb
CHANGED
@@ -1,6 +1,6 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
-
require "sidekiq/
|
3
|
+
require "sidekiq/job"
|
4
4
|
|
5
5
|
module Sidekiq
|
6
6
|
class Rails < ::Rails::Engine
|
@@ -33,19 +33,15 @@ module Sidekiq
|
|
33
33
|
# end
|
34
34
|
initializer "sidekiq.active_job_integration" do
|
35
35
|
ActiveSupport.on_load(:active_job) do
|
36
|
-
include ::Sidekiq::
|
36
|
+
include ::Sidekiq::Job::Options unless respond_to?(:sidekiq_options)
|
37
37
|
end
|
38
38
|
end
|
39
39
|
|
40
|
-
|
41
|
-
|
42
|
-
|
43
|
-
|
44
|
-
|
45
|
-
unless ::Rails.logger == ::Sidekiq.logger || ::ActiveSupport::Logger.logger_outputs_to?(::Rails.logger, $stdout)
|
46
|
-
::Rails.logger.extend(::ActiveSupport::Logger.broadcast(::Sidekiq.logger))
|
47
|
-
end
|
48
|
-
end
|
40
|
+
config.before_configuration do
|
41
|
+
dep = ActiveSupport::Deprecation.new("7.0", "Sidekiq")
|
42
|
+
dep.deprecate_methods(Sidekiq.singleton_class,
|
43
|
+
default_worker_options: :default_job_options,
|
44
|
+
"default_worker_options=": :default_job_options=)
|
49
45
|
end
|
50
46
|
|
51
47
|
# This hook happens after all initializers are run, just before returning
|
@@ -53,8 +49,18 @@ module Sidekiq
|
|
53
49
|
#
|
54
50
|
# None of this matters on the client-side, only within the Sidekiq process itself.
|
55
51
|
config.after_initialize do
|
56
|
-
Sidekiq.configure_server do |
|
57
|
-
|
52
|
+
Sidekiq.configure_server do |config|
|
53
|
+
config[:reloader] = Sidekiq::Rails::Reloader.new
|
54
|
+
|
55
|
+
# This is the integration code necessary so that if a job uses `Rails.logger.info "Hello"`,
|
56
|
+
# it will appear in the Sidekiq console with all of the job context.
|
57
|
+
unless ::Rails.logger == config.logger || ::ActiveSupport::Logger.logger_outputs_to?(::Rails.logger, $stdout)
|
58
|
+
if ::Rails::VERSION::STRING < "7.1"
|
59
|
+
::Rails.logger.extend(::ActiveSupport::Logger.broadcast(config.logger))
|
60
|
+
else
|
61
|
+
::Rails.logger.broadcast_to(config.logger)
|
62
|
+
end
|
63
|
+
end
|
58
64
|
end
|
59
65
|
end
|
60
66
|
end
|