sidekiq 6.0.0 → 6.5.7
Sign up to get free protection for your applications and to get access to all the features.
Potentially problematic release.
This version of sidekiq might be problematic. Click here for more details.
- checksums.yaml +4 -4
- data/Changes.md +383 -2
- data/LICENSE +3 -3
- data/README.md +13 -10
- data/bin/sidekiq +27 -3
- data/bin/sidekiqload +74 -66
- data/bin/sidekiqmon +5 -6
- data/lib/generators/sidekiq/job_generator.rb +57 -0
- data/lib/generators/sidekiq/templates/{worker.rb.erb → job.rb.erb} +2 -2
- data/lib/generators/sidekiq/templates/{worker_spec.rb.erb → job_spec.rb.erb} +1 -1
- data/lib/generators/sidekiq/templates/{worker_test.rb.erb → job_test.rb.erb} +1 -1
- data/lib/sidekiq/api.rb +446 -221
- data/lib/sidekiq/cli.rb +112 -63
- data/lib/sidekiq/client.rb +57 -60
- data/lib/sidekiq/{util.rb → component.rb} +12 -16
- data/lib/sidekiq/delay.rb +3 -1
- data/lib/sidekiq/extensions/action_mailer.rb +3 -2
- data/lib/sidekiq/extensions/active_record.rb +4 -3
- data/lib/sidekiq/extensions/class_methods.rb +5 -4
- data/lib/sidekiq/extensions/generic_proxy.rb +4 -2
- data/lib/sidekiq/fetch.rb +48 -37
- data/lib/sidekiq/job.rb +13 -0
- data/lib/sidekiq/job_logger.rb +19 -23
- data/lib/sidekiq/job_retry.rb +100 -67
- data/lib/sidekiq/job_util.rb +71 -0
- data/lib/sidekiq/launcher.rb +145 -59
- data/lib/sidekiq/logger.rb +99 -12
- data/lib/sidekiq/manager.rb +35 -34
- data/lib/sidekiq/metrics/deploy.rb +47 -0
- data/lib/sidekiq/metrics/query.rb +153 -0
- data/lib/sidekiq/metrics/shared.rb +94 -0
- data/lib/sidekiq/metrics/tracking.rb +134 -0
- data/lib/sidekiq/middleware/chain.rb +99 -44
- data/lib/sidekiq/middleware/current_attributes.rb +63 -0
- data/lib/sidekiq/middleware/i18n.rb +6 -4
- data/lib/sidekiq/middleware/modules.rb +21 -0
- data/lib/sidekiq/monitor.rb +4 -19
- data/lib/sidekiq/paginator.rb +13 -8
- data/lib/sidekiq/processor.rb +64 -60
- data/lib/sidekiq/rails.rb +38 -22
- data/lib/sidekiq/redis_client_adapter.rb +154 -0
- data/lib/sidekiq/redis_connection.rb +91 -54
- data/lib/sidekiq/ring_buffer.rb +29 -0
- data/lib/sidekiq/scheduled.rb +93 -28
- data/lib/sidekiq/sd_notify.rb +149 -0
- data/lib/sidekiq/systemd.rb +24 -0
- data/lib/sidekiq/testing/inline.rb +4 -4
- data/lib/sidekiq/testing.rb +51 -40
- data/lib/sidekiq/transaction_aware_client.rb +45 -0
- data/lib/sidekiq/version.rb +1 -1
- data/lib/sidekiq/web/action.rb +3 -3
- data/lib/sidekiq/web/application.rb +57 -34
- data/lib/sidekiq/web/csrf_protection.rb +180 -0
- data/lib/sidekiq/web/helpers.rb +77 -36
- data/lib/sidekiq/web/router.rb +6 -5
- data/lib/sidekiq/web.rb +41 -73
- data/lib/sidekiq/worker.rb +144 -21
- data/lib/sidekiq.rb +129 -32
- data/sidekiq.gemspec +14 -7
- data/web/assets/images/apple-touch-icon.png +0 -0
- data/web/assets/javascripts/application.js +112 -61
- data/web/assets/javascripts/chart.min.js +13 -0
- data/web/assets/javascripts/chartjs-plugin-annotation.min.js +7 -0
- data/web/assets/javascripts/dashboard.js +52 -69
- data/web/assets/javascripts/graph.js +16 -0
- data/web/assets/javascripts/metrics.js +262 -0
- data/web/assets/stylesheets/application-dark.css +143 -0
- data/web/assets/stylesheets/application-rtl.css +0 -4
- data/web/assets/stylesheets/application.css +88 -233
- data/web/locales/ar.yml +8 -2
- data/web/locales/de.yml +14 -2
- data/web/locales/el.yml +43 -19
- data/web/locales/en.yml +13 -1
- data/web/locales/es.yml +18 -2
- data/web/locales/fr.yml +10 -3
- data/web/locales/ja.yml +12 -0
- data/web/locales/lt.yml +83 -0
- data/web/locales/pl.yml +4 -4
- data/web/locales/pt-br.yml +27 -9
- data/web/locales/ru.yml +4 -0
- data/web/locales/vi.yml +83 -0
- data/web/locales/zh-cn.yml +36 -11
- data/web/locales/zh-tw.yml +32 -7
- data/web/views/_footer.erb +1 -1
- data/web/views/_job_info.erb +3 -2
- data/web/views/_nav.erb +1 -1
- data/web/views/_poll_link.erb +2 -5
- data/web/views/_summary.erb +7 -7
- data/web/views/busy.erb +56 -22
- data/web/views/dashboard.erb +23 -14
- data/web/views/dead.erb +3 -3
- data/web/views/layout.erb +3 -1
- data/web/views/metrics.erb +69 -0
- data/web/views/metrics_for_job.erb +87 -0
- data/web/views/morgue.erb +9 -6
- data/web/views/queue.erb +23 -10
- data/web/views/queues.erb +10 -2
- data/web/views/retries.erb +11 -8
- data/web/views/retry.erb +3 -3
- data/web/views/scheduled.erb +5 -2
- metadata +57 -58
- data/.circleci/config.yml +0 -61
- data/.github/contributing.md +0 -32
- data/.github/issue_template.md +0 -11
- data/.gitignore +0 -13
- data/.standard.yml +0 -20
- data/3.0-Upgrade.md +0 -70
- data/4.0-Upgrade.md +0 -53
- data/5.0-Upgrade.md +0 -56
- data/6.0-Upgrade.md +0 -70
- data/COMM-LICENSE +0 -97
- data/Ent-2.0-Upgrade.md +0 -37
- data/Ent-Changes.md +0 -250
- data/Gemfile +0 -24
- data/Gemfile.lock +0 -196
- data/Pro-2.0-Upgrade.md +0 -138
- data/Pro-3.0-Upgrade.md +0 -44
- data/Pro-4.0-Upgrade.md +0 -35
- data/Pro-5.0-Upgrade.md +0 -25
- data/Pro-Changes.md +0 -768
- data/Rakefile +0 -10
- data/code_of_conduct.md +0 -50
- data/lib/generators/sidekiq/worker_generator.rb +0 -47
- data/lib/sidekiq/exception_handler.rb +0 -27
data/lib/sidekiq/web/helpers.rb
CHANGED
@@ -10,18 +10,25 @@ module Sidekiq
|
|
10
10
|
module WebHelpers
|
11
11
|
def strings(lang)
|
12
12
|
@strings ||= {}
|
13
|
-
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
end
|
13
|
+
|
14
|
+
# Allow sidekiq-web extensions to add locale paths
|
15
|
+
# so extensions can be localized
|
16
|
+
@strings[lang] ||= settings.locales.each_with_object({}) do |path, global|
|
17
|
+
find_locale_files(lang).each do |file|
|
18
|
+
strs = YAML.safe_load(File.open(file))
|
19
|
+
global.merge!(strs[lang])
|
21
20
|
end
|
22
21
|
end
|
23
22
|
end
|
24
23
|
|
24
|
+
def singularize(str, count)
|
25
|
+
if count == 1 && str.respond_to?(:singularize) # rails
|
26
|
+
str.singularize
|
27
|
+
else
|
28
|
+
str
|
29
|
+
end
|
30
|
+
end
|
31
|
+
|
25
32
|
def clear_caches
|
26
33
|
@strings = nil
|
27
34
|
@locale_files = nil
|
@@ -63,14 +70,6 @@ module Sidekiq
|
|
63
70
|
@head_html.join if defined?(@head_html)
|
64
71
|
end
|
65
72
|
|
66
|
-
def poll_path
|
67
|
-
if current_path != "" && params["poll"]
|
68
|
-
root_path + current_path
|
69
|
-
else
|
70
|
-
""
|
71
|
-
end
|
72
|
-
end
|
73
|
-
|
74
73
|
def text_direction
|
75
74
|
get_locale["TextDirection"] || "ltr"
|
76
75
|
end
|
@@ -112,6 +111,13 @@ module Sidekiq
|
|
112
111
|
end
|
113
112
|
end
|
114
113
|
|
114
|
+
# within is used by Sidekiq Pro
|
115
|
+
def display_tags(job, within = nil)
|
116
|
+
job.tags.map { |tag|
|
117
|
+
"<span class='label label-info jobtag'>#{::Rack::Utils.escape_html(tag)}</span>"
|
118
|
+
}.join(" ")
|
119
|
+
end
|
120
|
+
|
115
121
|
# mperham/sidekiq#3243
|
116
122
|
def unfiltered?
|
117
123
|
yield unless env["PATH_INFO"].start_with?("/filter/")
|
@@ -130,28 +136,48 @@ module Sidekiq
|
|
130
136
|
end
|
131
137
|
end
|
132
138
|
|
133
|
-
def
|
134
|
-
|
139
|
+
def sort_direction_label
|
140
|
+
params[:direction] == "asc" ? "↑" : "↓"
|
141
|
+
end
|
142
|
+
|
143
|
+
def workset
|
144
|
+
@work ||= Sidekiq::WorkSet.new
|
135
145
|
end
|
136
146
|
|
137
147
|
def processes
|
138
148
|
@processes ||= Sidekiq::ProcessSet.new
|
139
149
|
end
|
140
150
|
|
141
|
-
|
142
|
-
|
151
|
+
# Sorts processes by hostname following the natural sort order so that
|
152
|
+
# 'worker.1' < 'worker.2' < 'worker.10' < 'worker.20'
|
153
|
+
# '2.1.1.1' < '192.168.0.2' < '192.168.0.10'
|
154
|
+
def sorted_processes
|
155
|
+
@sorted_processes ||= begin
|
156
|
+
return processes unless processes.all? { |p| p["hostname"] }
|
157
|
+
|
158
|
+
split_characters = /[._-]+/
|
159
|
+
|
160
|
+
padding = processes.flat_map { |p| p["hostname"].split(split_characters) }.map(&:size).max
|
161
|
+
|
162
|
+
processes.to_a.sort_by do |process|
|
163
|
+
process["hostname"].split(split_characters).map do |substring|
|
164
|
+
# Left-pad the substring with '0' if it starts with a number or 'a'
|
165
|
+
# otherwise, so that '25' < 192' < 'a' ('025' < '192' < 'aaa')
|
166
|
+
padding_char = substring[0].match?(/\d/) ? "0" : "a"
|
167
|
+
|
168
|
+
substring.rjust(padding, padding_char)
|
169
|
+
end
|
170
|
+
end
|
171
|
+
end
|
143
172
|
end
|
144
173
|
|
145
|
-
def
|
146
|
-
Sidekiq.
|
147
|
-
conn.zrangebyscore("retry", score, score)
|
148
|
-
}.map { |msg| Sidekiq.load_json(msg) }
|
174
|
+
def stats
|
175
|
+
@stats ||= Sidekiq::Stats.new
|
149
176
|
end
|
150
177
|
|
151
178
|
def redis_connection
|
152
179
|
Sidekiq.redis do |conn|
|
153
|
-
|
154
|
-
"redis://#{c[:location]}/#{c[:db]}"
|
180
|
+
conn.connection[:id]
|
155
181
|
end
|
156
182
|
end
|
157
183
|
|
@@ -172,7 +198,7 @@ module Sidekiq
|
|
172
198
|
end
|
173
199
|
|
174
200
|
def current_status
|
175
|
-
|
201
|
+
workset.size == 0 ? "idle" : "active"
|
176
202
|
end
|
177
203
|
|
178
204
|
def relative_time(time)
|
@@ -189,16 +215,17 @@ module Sidekiq
|
|
189
215
|
[score.to_f, jid]
|
190
216
|
end
|
191
217
|
|
192
|
-
SAFE_QPARAMS = %w[page
|
218
|
+
SAFE_QPARAMS = %w[page direction]
|
193
219
|
|
194
220
|
# Merge options with current params, filter safe params, and stringify to query string
|
195
221
|
def qparams(options)
|
196
|
-
|
197
|
-
options.keys.each do |key|
|
198
|
-
options[key.to_s] = options.delete(key)
|
199
|
-
end
|
222
|
+
stringified_options = options.transform_keys(&:to_s)
|
200
223
|
|
201
|
-
params.merge(
|
224
|
+
to_query_string(params.merge(stringified_options))
|
225
|
+
end
|
226
|
+
|
227
|
+
def to_query_string(params)
|
228
|
+
params.map { |key, value|
|
202
229
|
SAFE_QPARAMS.include?(key) ? "#{key}=#{CGI.escape(value.to_s)}" : next
|
203
230
|
}.compact.join("&")
|
204
231
|
end
|
@@ -221,7 +248,7 @@ module Sidekiq
|
|
221
248
|
end
|
222
249
|
|
223
250
|
def csrf_tag
|
224
|
-
"<input type='hidden' name='authenticity_token' value='#{
|
251
|
+
"<input type='hidden' name='authenticity_token' value='#{env[:csrf_token]}'/>"
|
225
252
|
end
|
226
253
|
|
227
254
|
def to_display(arg)
|
@@ -238,7 +265,7 @@ module Sidekiq
|
|
238
265
|
queue class args retry_count retried_at failed_at
|
239
266
|
jid error_message error_class backtrace
|
240
267
|
error_backtrace enqueued_at retry wrapped
|
241
|
-
created_at
|
268
|
+
created_at tags display_class
|
242
269
|
])
|
243
270
|
|
244
271
|
def retry_extra_items(retry_job)
|
@@ -249,7 +276,21 @@ module Sidekiq
|
|
249
276
|
end
|
250
277
|
end
|
251
278
|
|
279
|
+
def format_memory(rss_kb)
|
280
|
+
return "0" if rss_kb.nil? || rss_kb == 0
|
281
|
+
|
282
|
+
if rss_kb < 100_000
|
283
|
+
"#{number_with_delimiter(rss_kb)} KB"
|
284
|
+
elsif rss_kb < 10_000_000
|
285
|
+
"#{number_with_delimiter((rss_kb / 1024.0).to_i)} MB"
|
286
|
+
else
|
287
|
+
"#{number_with_delimiter((rss_kb / (1024.0 * 1024.0)).round(1))} GB"
|
288
|
+
end
|
289
|
+
end
|
290
|
+
|
252
291
|
def number_with_delimiter(number)
|
292
|
+
return "" if number.nil?
|
293
|
+
|
253
294
|
begin
|
254
295
|
Float(number)
|
255
296
|
rescue ArgumentError, TypeError
|
@@ -283,7 +324,7 @@ module Sidekiq
|
|
283
324
|
end
|
284
325
|
|
285
326
|
def environment_title_prefix
|
286
|
-
environment = Sidekiq
|
327
|
+
environment = Sidekiq[:environment] || ENV["APP_ENV"] || ENV["RAILS_ENV"] || ENV["RACK_ENV"] || "development"
|
287
328
|
|
288
329
|
"[#{environment.upcase}] " unless environment == "production"
|
289
330
|
end
|
data/lib/sidekiq/web/router.rb
CHANGED
@@ -15,6 +15,10 @@ module Sidekiq
|
|
15
15
|
REQUEST_METHOD = "REQUEST_METHOD"
|
16
16
|
PATH_INFO = "PATH_INFO"
|
17
17
|
|
18
|
+
def head(path, &block)
|
19
|
+
route(HEAD, path, &block)
|
20
|
+
end
|
21
|
+
|
18
22
|
def get(path, &block)
|
19
23
|
route(GET, path, &block)
|
20
24
|
end
|
@@ -39,7 +43,6 @@ module Sidekiq
|
|
39
43
|
@routes ||= {GET => [], POST => [], PUT => [], PATCH => [], DELETE => [], HEAD => []}
|
40
44
|
|
41
45
|
@routes[method] << WebRoute.new(method, path, block)
|
42
|
-
@routes[HEAD] << WebRoute.new(method, path, block) if method == GET
|
43
46
|
end
|
44
47
|
|
45
48
|
def match(env)
|
@@ -66,7 +69,7 @@ module Sidekiq
|
|
66
69
|
class WebRoute
|
67
70
|
attr_accessor :request_method, :pattern, :block, :name
|
68
71
|
|
69
|
-
NAMED_SEGMENTS_PATTERN = /\/([^\/]*):([
|
72
|
+
NAMED_SEGMENTS_PATTERN = /\/([^\/]*):([^.:$\/]+)/
|
70
73
|
|
71
74
|
def initialize(request_method, pattern, block)
|
72
75
|
@request_method = request_method
|
@@ -94,9 +97,7 @@ module Sidekiq
|
|
94
97
|
{} if path == matcher
|
95
98
|
else
|
96
99
|
path_match = path.match(matcher)
|
97
|
-
|
98
|
-
Hash[path_match.names.map(&:to_sym).zip(path_match.captures)]
|
99
|
-
end
|
100
|
+
path_match&.named_captures&.transform_keys(&:to_sym)
|
100
101
|
end
|
101
102
|
end
|
102
103
|
end
|
data/lib/sidekiq/web.rb
CHANGED
@@ -10,12 +10,11 @@ require "sidekiq/web/helpers"
|
|
10
10
|
require "sidekiq/web/router"
|
11
11
|
require "sidekiq/web/action"
|
12
12
|
require "sidekiq/web/application"
|
13
|
+
require "sidekiq/web/csrf_protection"
|
13
14
|
|
14
|
-
require "rack/
|
15
|
-
|
15
|
+
require "rack/content_length"
|
16
16
|
require "rack/builder"
|
17
|
-
require "rack/
|
18
|
-
require "rack/session/cookie"
|
17
|
+
require "rack/static"
|
19
18
|
|
20
19
|
module Sidekiq
|
21
20
|
class Web
|
@@ -31,22 +30,18 @@ module Sidekiq
|
|
31
30
|
"Queues" => "queues",
|
32
31
|
"Retries" => "retries",
|
33
32
|
"Scheduled" => "scheduled",
|
34
|
-
"Dead" => "morgue"
|
33
|
+
"Dead" => "morgue"
|
35
34
|
}
|
36
35
|
|
36
|
+
if ENV["SIDEKIQ_METRICS_BETA"] == "1"
|
37
|
+
DEFAULT_TABS["Metrics"] = "metrics"
|
38
|
+
end
|
39
|
+
|
37
40
|
class << self
|
38
41
|
def settings
|
39
42
|
self
|
40
43
|
end
|
41
44
|
|
42
|
-
def middlewares
|
43
|
-
@middlewares ||= []
|
44
|
-
end
|
45
|
-
|
46
|
-
def use(*middleware_args, &block)
|
47
|
-
middlewares << [middleware_args, block]
|
48
|
-
end
|
49
|
-
|
50
45
|
def default_tabs
|
51
46
|
DEFAULT_TABS
|
52
47
|
end
|
@@ -72,32 +67,45 @@ module Sidekiq
|
|
72
67
|
opts.each { |key| set(key, false) }
|
73
68
|
end
|
74
69
|
|
75
|
-
|
70
|
+
def middlewares
|
71
|
+
@middlewares ||= []
|
72
|
+
end
|
73
|
+
|
74
|
+
def use(*args, &block)
|
75
|
+
middlewares << [args, block]
|
76
|
+
end
|
77
|
+
|
76
78
|
def set(attribute, value)
|
77
79
|
send(:"#{attribute}=", value)
|
78
80
|
end
|
79
81
|
|
80
|
-
|
82
|
+
def sessions=(val)
|
83
|
+
puts "WARNING: Sidekiq::Web.sessions= is no longer relevant and will be removed in Sidekiq 7.0. #{caller(1..1).first}"
|
84
|
+
end
|
85
|
+
|
86
|
+
def session_secret=(val)
|
87
|
+
puts "WARNING: Sidekiq::Web.session_secret= is no longer relevant and will be removed in Sidekiq 7.0. #{caller(1..1).first}"
|
88
|
+
end
|
89
|
+
|
90
|
+
attr_accessor :app_url, :redis_pool
|
81
91
|
attr_writer :locales, :views
|
82
92
|
end
|
83
93
|
|
84
94
|
def self.inherited(child)
|
85
95
|
child.app_url = app_url
|
86
|
-
child.session_secret = session_secret
|
87
96
|
child.redis_pool = redis_pool
|
88
|
-
child.sessions = sessions
|
89
97
|
end
|
90
98
|
|
91
99
|
def settings
|
92
100
|
self.class.settings
|
93
101
|
end
|
94
102
|
|
95
|
-
def
|
96
|
-
middlewares
|
103
|
+
def middlewares
|
104
|
+
@middlewares ||= self.class.middlewares
|
97
105
|
end
|
98
106
|
|
99
|
-
def
|
100
|
-
|
107
|
+
def use(*args, &block)
|
108
|
+
middlewares << [args, block]
|
101
109
|
end
|
102
110
|
|
103
111
|
def call(env)
|
@@ -125,18 +133,8 @@ module Sidekiq
|
|
125
133
|
send(:"#{attribute}=", value)
|
126
134
|
end
|
127
135
|
|
128
|
-
|
129
|
-
|
130
|
-
|
131
|
-
attr_writer :sessions
|
132
|
-
|
133
|
-
def sessions
|
134
|
-
unless instance_variable_defined?("@sessions")
|
135
|
-
@sessions = self.class.sessions
|
136
|
-
@sessions = @sessions.to_hash.dup if @sessions.respond_to?(:to_hash)
|
137
|
-
end
|
138
|
-
|
139
|
-
@sessions
|
136
|
+
def sessions=(val)
|
137
|
+
puts "Sidekiq::Web#sessions= is no longer relevant and will be removed in Sidekiq 7.0. #{caller[2..2].first}"
|
140
138
|
end
|
141
139
|
|
142
140
|
def self.register(extension)
|
@@ -145,50 +143,20 @@ module Sidekiq
|
|
145
143
|
|
146
144
|
private
|
147
145
|
|
148
|
-
def using?(middleware)
|
149
|
-
middlewares.any? do |(m, _)|
|
150
|
-
m.is_a?(Array) && (m[0] == middleware || m[0].is_a?(middleware))
|
151
|
-
end
|
152
|
-
end
|
153
|
-
|
154
|
-
def build_sessions
|
155
|
-
middlewares = self.middlewares
|
156
|
-
|
157
|
-
unless using?(::Rack::Protection) || ENV["RACK_ENV"] == "test"
|
158
|
-
middlewares.unshift [[::Rack::Protection, {use: :authenticity_token}], nil]
|
159
|
-
end
|
160
|
-
|
161
|
-
s = sessions
|
162
|
-
return unless s
|
163
|
-
|
164
|
-
unless using? ::Rack::Session::Cookie
|
165
|
-
unless (secret = Web.session_secret)
|
166
|
-
require "securerandom"
|
167
|
-
secret = SecureRandom.hex(64)
|
168
|
-
end
|
169
|
-
|
170
|
-
options = {secret: secret}
|
171
|
-
options = options.merge(s.to_hash) if s.respond_to? :to_hash
|
172
|
-
|
173
|
-
middlewares.unshift [[::Rack::Session::Cookie, options], nil]
|
174
|
-
end
|
175
|
-
end
|
176
|
-
|
177
146
|
def build
|
178
|
-
build_sessions
|
179
|
-
|
180
|
-
middlewares = self.middlewares
|
181
147
|
klass = self.class
|
148
|
+
m = middlewares
|
182
149
|
|
183
|
-
|
184
|
-
|
185
|
-
map "/#{asset_dir}" do
|
186
|
-
run ::Rack::File.new("#{ASSETS}/#{asset_dir}", {"Cache-Control" => "public, max-age=86400"})
|
187
|
-
end
|
188
|
-
end
|
189
|
-
|
190
|
-
middlewares.each { |middleware, block| use(*middleware, &block) }
|
150
|
+
rules = []
|
151
|
+
rules = [[:all, {"cache-control" => "public, max-age=86400"}]] unless ENV["SIDEKIQ_WEB_TESTING"]
|
191
152
|
|
153
|
+
::Rack::Builder.new do
|
154
|
+
use Rack::Static, urls: ["/stylesheets", "/images", "/javascripts"],
|
155
|
+
root: ASSETS,
|
156
|
+
cascade: true,
|
157
|
+
header_rules: rules
|
158
|
+
m.each { |middleware, block| use(*middleware, &block) }
|
159
|
+
use Sidekiq::Web::CsrfProtection unless $TESTING
|
192
160
|
run WebApplication.new(klass)
|
193
161
|
end
|
194
162
|
end
|
data/lib/sidekiq/worker.rb
CHANGED
@@ -9,6 +9,7 @@ module Sidekiq
|
|
9
9
|
#
|
10
10
|
# class HardWorker
|
11
11
|
# include Sidekiq::Worker
|
12
|
+
# sidekiq_options queue: 'critical', retry: 5
|
12
13
|
#
|
13
14
|
# def perform(*args)
|
14
15
|
# # do some work
|
@@ -20,6 +21,26 @@ module Sidekiq
|
|
20
21
|
# HardWorker.perform_async(1, 2, 3)
|
21
22
|
#
|
22
23
|
# Note that perform_async is a class method, perform is an instance method.
|
24
|
+
#
|
25
|
+
# Sidekiq::Worker also includes several APIs to provide compatibility with
|
26
|
+
# ActiveJob.
|
27
|
+
#
|
28
|
+
# class SomeWorker
|
29
|
+
# include Sidekiq::Worker
|
30
|
+
# queue_as :critical
|
31
|
+
#
|
32
|
+
# def perform(...)
|
33
|
+
# end
|
34
|
+
# end
|
35
|
+
#
|
36
|
+
# SomeWorker.set(wait_until: 1.hour).perform_async(123)
|
37
|
+
#
|
38
|
+
# Note that arguments passed to the job must still obey Sidekiq's
|
39
|
+
# best practice for simple, JSON-native data types. Sidekiq will not
|
40
|
+
# implement ActiveJob's more complex argument serialization. For
|
41
|
+
# this reason, we don't implement `perform_later` as our call semantics
|
42
|
+
# are very different.
|
43
|
+
#
|
23
44
|
module Worker
|
24
45
|
##
|
25
46
|
# The Options module is extracted so we can include it in ActiveJob::Base
|
@@ -48,8 +69,8 @@ module Sidekiq
|
|
48
69
|
# In practice, any option is allowed. This is the main mechanism to configure the
|
49
70
|
# options for a specific job.
|
50
71
|
def sidekiq_options(opts = {})
|
51
|
-
opts =
|
52
|
-
self.sidekiq_options_hash = get_sidekiq_options.merge(
|
72
|
+
opts = opts.transform_keys(&:to_s) # stringify
|
73
|
+
self.sidekiq_options_hash = get_sidekiq_options.merge(opts)
|
53
74
|
end
|
54
75
|
|
55
76
|
def sidekiq_retry_in(&block)
|
@@ -61,7 +82,7 @@ module Sidekiq
|
|
61
82
|
end
|
62
83
|
|
63
84
|
def get_sidekiq_options # :nodoc:
|
64
|
-
self.sidekiq_options_hash ||= Sidekiq.
|
85
|
+
self.sidekiq_options_hash ||= Sidekiq.default_job_options
|
65
86
|
end
|
66
87
|
|
67
88
|
def sidekiq_class_attribute(*attrs)
|
@@ -150,33 +171,97 @@ module Sidekiq
|
|
150
171
|
# SomeWorker.set(queue: 'foo').perform_async(....)
|
151
172
|
#
|
152
173
|
class Setter
|
174
|
+
include Sidekiq::JobUtil
|
175
|
+
|
153
176
|
def initialize(klass, opts)
|
154
177
|
@klass = klass
|
155
|
-
|
178
|
+
# NB: the internal hash always has stringified keys
|
179
|
+
@opts = opts.transform_keys(&:to_s)
|
180
|
+
|
181
|
+
# ActiveJob compatibility
|
182
|
+
interval = @opts.delete("wait_until") || @opts.delete("wait")
|
183
|
+
at(interval) if interval
|
156
184
|
end
|
157
185
|
|
158
186
|
def set(options)
|
159
|
-
|
187
|
+
hash = options.transform_keys(&:to_s)
|
188
|
+
interval = hash.delete("wait_until") || @opts.delete("wait")
|
189
|
+
@opts.merge!(hash)
|
190
|
+
at(interval) if interval
|
160
191
|
self
|
161
192
|
end
|
162
193
|
|
163
194
|
def perform_async(*args)
|
164
|
-
@
|
195
|
+
if @opts["sync"] == true
|
196
|
+
perform_inline(*args)
|
197
|
+
else
|
198
|
+
@klass.client_push(@opts.merge("args" => args, "class" => @klass))
|
199
|
+
end
|
200
|
+
end
|
201
|
+
|
202
|
+
# Explicit inline execution of a job. Returns nil if the job did not
|
203
|
+
# execute, true otherwise.
|
204
|
+
def perform_inline(*args)
|
205
|
+
raw = @opts.merge("args" => args, "class" => @klass)
|
206
|
+
|
207
|
+
# validate and normalize payload
|
208
|
+
item = normalize_item(raw)
|
209
|
+
queue = item["queue"]
|
210
|
+
|
211
|
+
# run client-side middleware
|
212
|
+
result = Sidekiq.client_middleware.invoke(item["class"], item, queue, Sidekiq.redis_pool) do
|
213
|
+
item
|
214
|
+
end
|
215
|
+
return nil unless result
|
216
|
+
|
217
|
+
# round-trip the payload via JSON
|
218
|
+
msg = Sidekiq.load_json(Sidekiq.dump_json(item))
|
219
|
+
|
220
|
+
# prepare the job instance
|
221
|
+
klass = msg["class"].constantize
|
222
|
+
job = klass.new
|
223
|
+
job.jid = msg["jid"]
|
224
|
+
job.bid = msg["bid"] if job.respond_to?(:bid)
|
225
|
+
|
226
|
+
# run the job through server-side middleware
|
227
|
+
result = Sidekiq.server_middleware.invoke(job, msg, msg["queue"]) do
|
228
|
+
# perform it
|
229
|
+
job.perform(*msg["args"])
|
230
|
+
true
|
231
|
+
end
|
232
|
+
return nil unless result
|
233
|
+
# jobs do not return a result. they should store any
|
234
|
+
# modified state.
|
235
|
+
true
|
236
|
+
end
|
237
|
+
alias_method :perform_sync, :perform_inline
|
238
|
+
|
239
|
+
def perform_bulk(args, batch_size: 1_000)
|
240
|
+
client = @klass.build_client
|
241
|
+
result = args.each_slice(batch_size).flat_map do |slice|
|
242
|
+
client.push_bulk(@opts.merge("class" => @klass, "args" => slice))
|
243
|
+
end
|
244
|
+
|
245
|
+
result.is_a?(Enumerator::Lazy) ? result.force : result
|
165
246
|
end
|
166
247
|
|
167
248
|
# +interval+ must be a timestamp, numeric or something that acts
|
168
249
|
# numeric (like an activesupport time interval).
|
169
250
|
def perform_in(interval, *args)
|
251
|
+
at(interval).perform_async(*args)
|
252
|
+
end
|
253
|
+
alias_method :perform_at, :perform_in
|
254
|
+
|
255
|
+
private
|
256
|
+
|
257
|
+
def at(interval)
|
170
258
|
int = interval.to_f
|
171
259
|
now = Time.now.to_f
|
172
260
|
ts = (int < 1_000_000_000 ? now + int : int)
|
173
|
-
|
174
|
-
payload = @opts.merge("class" => @klass, "args" => args, "at" => ts)
|
175
261
|
# Optimization to enqueue something now that is scheduled to go out now or in the past
|
176
|
-
|
177
|
-
|
262
|
+
@opts["at"] = ts if ts > now
|
263
|
+
self
|
178
264
|
end
|
179
|
-
alias_method :perform_at, :perform_in
|
180
265
|
end
|
181
266
|
|
182
267
|
module ClassMethods
|
@@ -192,12 +277,46 @@ module Sidekiq
|
|
192
277
|
raise ArgumentError, "Do not call .delay_until on a Sidekiq::Worker class, call .perform_at"
|
193
278
|
end
|
194
279
|
|
280
|
+
def queue_as(q)
|
281
|
+
sidekiq_options("queue" => q.to_s)
|
282
|
+
end
|
283
|
+
|
195
284
|
def set(options)
|
196
285
|
Setter.new(self, options)
|
197
286
|
end
|
198
287
|
|
199
288
|
def perform_async(*args)
|
200
|
-
|
289
|
+
Setter.new(self, {}).perform_async(*args)
|
290
|
+
end
|
291
|
+
|
292
|
+
# Inline execution of job's perform method after passing through Sidekiq.client_middleware and Sidekiq.server_middleware
|
293
|
+
def perform_inline(*args)
|
294
|
+
Setter.new(self, {}).perform_inline(*args)
|
295
|
+
end
|
296
|
+
alias_method :perform_sync, :perform_inline
|
297
|
+
|
298
|
+
##
|
299
|
+
# Push a large number of jobs to Redis, while limiting the batch of
|
300
|
+
# each job payload to 1,000. This method helps cut down on the number
|
301
|
+
# of round trips to Redis, which can increase the performance of enqueueing
|
302
|
+
# large numbers of jobs.
|
303
|
+
#
|
304
|
+
# +items+ must be an Array of Arrays.
|
305
|
+
#
|
306
|
+
# For finer-grained control, use `Sidekiq::Client.push_bulk` directly.
|
307
|
+
#
|
308
|
+
# Example (3 Redis round trips):
|
309
|
+
#
|
310
|
+
# SomeWorker.perform_async(1)
|
311
|
+
# SomeWorker.perform_async(2)
|
312
|
+
# SomeWorker.perform_async(3)
|
313
|
+
#
|
314
|
+
# Would instead become (1 Redis round trip):
|
315
|
+
#
|
316
|
+
# SomeWorker.perform_bulk([[1], [2], [3]])
|
317
|
+
#
|
318
|
+
def perform_bulk(*args, **kwargs)
|
319
|
+
Setter.new(self, {}).perform_bulk(*args, **kwargs)
|
201
320
|
end
|
202
321
|
|
203
322
|
# +interval+ must be a timestamp, numeric or something that acts
|
@@ -207,10 +326,10 @@ module Sidekiq
|
|
207
326
|
now = Time.now.to_f
|
208
327
|
ts = (int < 1_000_000_000 ? now + int : int)
|
209
328
|
|
210
|
-
item = {"class" => self, "args" => args
|
329
|
+
item = {"class" => self, "args" => args}
|
211
330
|
|
212
331
|
# Optimization to enqueue something now that is scheduled to go out now or in the past
|
213
|
-
item
|
332
|
+
item["at"] = ts if ts > now
|
214
333
|
|
215
334
|
client_push(item)
|
216
335
|
end
|
@@ -221,7 +340,7 @@ module Sidekiq
|
|
221
340
|
# Legal options:
|
222
341
|
#
|
223
342
|
# queue - use a named queue for this Worker, default 'default'
|
224
|
-
# retry - enable
|
343
|
+
# retry - enable retries via JobRetry, *true* to use the default
|
225
344
|
# or *Integer* count
|
226
345
|
# backtrace - whether to save any error backtrace in the retry payload to display in web UI,
|
227
346
|
# can be true, false or an integer number of lines to save, default *false*
|
@@ -229,18 +348,22 @@ module Sidekiq
|
|
229
348
|
#
|
230
349
|
# In practice, any option is allowed. This is the main mechanism to configure the
|
231
350
|
# options for a specific job.
|
351
|
+
#
|
352
|
+
# These options will be saved into the serialized job when enqueued by
|
353
|
+
# the client.
|
232
354
|
def sidekiq_options(opts = {})
|
233
355
|
super
|
234
356
|
end
|
235
357
|
|
236
358
|
def client_push(item) # :nodoc:
|
237
|
-
|
238
|
-
|
239
|
-
|
240
|
-
item[key.to_s] = item.delete(key)
|
241
|
-
end
|
359
|
+
raise ArgumentError, "Job payloads should contain no Symbols: #{item}" if item.any? { |k, v| k.is_a?(::Symbol) }
|
360
|
+
build_client.push(item)
|
361
|
+
end
|
242
362
|
|
243
|
-
|
363
|
+
def build_client # :nodoc:
|
364
|
+
pool = Thread.current[:sidekiq_via_pool] || get_sidekiq_options["pool"] || Sidekiq.redis_pool
|
365
|
+
client_class = get_sidekiq_options["client_class"] || Sidekiq::Client
|
366
|
+
client_class.new(pool)
|
244
367
|
end
|
245
368
|
end
|
246
369
|
end
|