sidekiq 6.2.1 → 6.5.1
Sign up to get free protection for your applications and to get access to all the features.
Potentially problematic release.
This version of sidekiq might be problematic. Click here for more details.
- checksums.yaml +4 -4
- data/Changes.md +132 -1
- data/LICENSE +3 -3
- data/README.md +9 -4
- data/bin/sidekiq +3 -3
- data/bin/sidekiqload +70 -66
- data/bin/sidekiqmon +1 -1
- data/lib/generators/sidekiq/job_generator.rb +57 -0
- data/lib/generators/sidekiq/templates/{worker.rb.erb → job.rb.erb} +2 -2
- data/lib/generators/sidekiq/templates/{worker_spec.rb.erb → job_spec.rb.erb} +1 -1
- data/lib/generators/sidekiq/templates/{worker_test.rb.erb → job_test.rb.erb} +1 -1
- data/lib/sidekiq/.DS_Store +0 -0
- data/lib/sidekiq/api.rb +192 -135
- data/lib/sidekiq/cli.rb +59 -40
- data/lib/sidekiq/client.rb +46 -66
- data/lib/sidekiq/{util.rb → component.rb} +11 -42
- data/lib/sidekiq/delay.rb +3 -1
- data/lib/sidekiq/extensions/generic_proxy.rb +4 -2
- data/lib/sidekiq/fetch.rb +23 -20
- data/lib/sidekiq/job.rb +13 -0
- data/lib/sidekiq/job_logger.rb +16 -28
- data/lib/sidekiq/job_retry.rb +37 -38
- data/lib/sidekiq/job_util.rb +71 -0
- data/lib/sidekiq/launcher.rb +67 -65
- data/lib/sidekiq/logger.rb +8 -18
- data/lib/sidekiq/manager.rb +35 -34
- data/lib/sidekiq/middleware/chain.rb +27 -16
- data/lib/sidekiq/middleware/current_attributes.rb +61 -0
- data/lib/sidekiq/middleware/i18n.rb +6 -4
- data/lib/sidekiq/middleware/modules.rb +21 -0
- data/lib/sidekiq/monitor.rb +1 -1
- data/lib/sidekiq/paginator.rb +8 -8
- data/lib/sidekiq/processor.rb +38 -38
- data/lib/sidekiq/rails.rb +22 -4
- data/lib/sidekiq/redis_client_adapter.rb +154 -0
- data/lib/sidekiq/redis_connection.rb +85 -54
- data/lib/sidekiq/ring_buffer.rb +29 -0
- data/lib/sidekiq/scheduled.rb +60 -24
- data/lib/sidekiq/testing/inline.rb +4 -4
- data/lib/sidekiq/testing.rb +38 -39
- data/lib/sidekiq/transaction_aware_client.rb +45 -0
- data/lib/sidekiq/version.rb +1 -1
- data/lib/sidekiq/web/action.rb +1 -1
- data/lib/sidekiq/web/application.rb +9 -6
- data/lib/sidekiq/web/csrf_protection.rb +2 -2
- data/lib/sidekiq/web/helpers.rb +14 -26
- data/lib/sidekiq/web.rb +6 -5
- data/lib/sidekiq/worker.rb +136 -13
- data/lib/sidekiq.rb +105 -30
- data/sidekiq.gemspec +1 -1
- data/web/assets/javascripts/application.js +113 -60
- data/web/assets/javascripts/dashboard.js +51 -51
- data/web/assets/stylesheets/application-dark.css +28 -45
- data/web/assets/stylesheets/application-rtl.css +0 -4
- data/web/assets/stylesheets/application.css +24 -237
- data/web/locales/ar.yml +8 -2
- data/web/locales/en.yml +4 -1
- data/web/locales/es.yml +18 -2
- data/web/locales/fr.yml +7 -0
- data/web/locales/ja.yml +3 -0
- data/web/locales/lt.yml +1 -1
- data/web/locales/pt-br.yml +27 -9
- data/web/views/_footer.erb +1 -1
- data/web/views/_job_info.erb +1 -1
- data/web/views/_poll_link.erb +2 -5
- data/web/views/_summary.erb +7 -7
- data/web/views/busy.erb +8 -8
- data/web/views/dashboard.erb +22 -14
- data/web/views/dead.erb +1 -1
- data/web/views/layout.erb +1 -1
- data/web/views/morgue.erb +6 -6
- data/web/views/queue.erb +10 -10
- data/web/views/queues.erb +3 -3
- data/web/views/retries.erb +7 -7
- data/web/views/retry.erb +1 -1
- data/web/views/scheduled.erb +1 -1
- metadata +17 -10
- data/lib/generators/sidekiq/worker_generator.rb +0 -57
- data/lib/sidekiq/exception_handler.rb +0 -27
data/lib/sidekiq/testing.rb
CHANGED
@@ -101,20 +101,20 @@ module Sidekiq
|
|
101
101
|
##
|
102
102
|
# The Queues class is only for testing the fake queue implementation.
|
103
103
|
# There are 2 data structures involved in tandem. This is due to the
|
104
|
-
# Rspec syntax of change(
|
104
|
+
# Rspec syntax of change(HardJob.jobs, :size). It keeps a reference
|
105
105
|
# to the array. Because the array was dervied from a filter of the total
|
106
106
|
# jobs enqueued, it appeared as though the array didn't change.
|
107
107
|
#
|
108
108
|
# To solve this, we'll keep 2 hashes containing the jobs. One with keys based
|
109
|
-
# on the queue, and another with keys of the
|
110
|
-
#
|
109
|
+
# on the queue, and another with keys of the job type, so the array for
|
110
|
+
# HardJob.jobs is a straight reference to a real array.
|
111
111
|
#
|
112
112
|
# Queue-based hash:
|
113
113
|
#
|
114
114
|
# {
|
115
115
|
# "default"=>[
|
116
116
|
# {
|
117
|
-
# "class"=>"TestTesting::
|
117
|
+
# "class"=>"TestTesting::HardJob",
|
118
118
|
# "args"=>[1, 2],
|
119
119
|
# "retry"=>true,
|
120
120
|
# "queue"=>"default",
|
@@ -124,12 +124,12 @@ module Sidekiq
|
|
124
124
|
# ]
|
125
125
|
# }
|
126
126
|
#
|
127
|
-
#
|
127
|
+
# Job-based hash:
|
128
128
|
#
|
129
129
|
# {
|
130
|
-
# "TestTesting::
|
130
|
+
# "TestTesting::HardJob"=>[
|
131
131
|
# {
|
132
|
-
# "class"=>"TestTesting::
|
132
|
+
# "class"=>"TestTesting::HardJob",
|
133
133
|
# "args"=>[1, 2],
|
134
134
|
# "retry"=>true,
|
135
135
|
# "queue"=>"default",
|
@@ -144,14 +144,14 @@ module Sidekiq
|
|
144
144
|
# require 'sidekiq/testing'
|
145
145
|
#
|
146
146
|
# assert_equal 0, Sidekiq::Queues["default"].size
|
147
|
-
#
|
147
|
+
# HardJob.perform_async(:something)
|
148
148
|
# assert_equal 1, Sidekiq::Queues["default"].size
|
149
149
|
# assert_equal :something, Sidekiq::Queues["default"].first['args'][0]
|
150
150
|
#
|
151
|
-
# You can also clear all
|
151
|
+
# You can also clear all jobs:
|
152
152
|
#
|
153
153
|
# assert_equal 0, Sidekiq::Queues["default"].size
|
154
|
-
#
|
154
|
+
# HardJob.perform_async(:something)
|
155
155
|
# Sidekiq::Queues.clear_all
|
156
156
|
# assert_equal 0, Sidekiq::Queues["default"].size
|
157
157
|
#
|
@@ -170,35 +170,36 @@ module Sidekiq
|
|
170
170
|
|
171
171
|
def push(queue, klass, job)
|
172
172
|
jobs_by_queue[queue] << job
|
173
|
-
|
173
|
+
jobs_by_class[klass] << job
|
174
174
|
end
|
175
175
|
|
176
176
|
def jobs_by_queue
|
177
177
|
@jobs_by_queue ||= Hash.new { |hash, key| hash[key] = [] }
|
178
178
|
end
|
179
179
|
|
180
|
-
def
|
181
|
-
@
|
180
|
+
def jobs_by_class
|
181
|
+
@jobs_by_class ||= Hash.new { |hash, key| hash[key] = [] }
|
182
182
|
end
|
183
|
+
alias_method :jobs_by_worker, :jobs_by_class
|
183
184
|
|
184
185
|
def delete_for(jid, queue, klass)
|
185
186
|
jobs_by_queue[queue.to_s].delete_if { |job| job["jid"] == jid }
|
186
|
-
|
187
|
+
jobs_by_class[klass].delete_if { |job| job["jid"] == jid }
|
187
188
|
end
|
188
189
|
|
189
190
|
def clear_for(queue, klass)
|
190
|
-
jobs_by_queue[queue].clear
|
191
|
-
|
191
|
+
jobs_by_queue[queue.to_s].clear
|
192
|
+
jobs_by_class[klass].clear
|
192
193
|
end
|
193
194
|
|
194
195
|
def clear_all
|
195
196
|
jobs_by_queue.clear
|
196
|
-
|
197
|
+
jobs_by_class.clear
|
197
198
|
end
|
198
199
|
end
|
199
200
|
end
|
200
201
|
|
201
|
-
module
|
202
|
+
module Job
|
202
203
|
##
|
203
204
|
# The Sidekiq testing infrastructure overrides perform_async
|
204
205
|
# so that it does not actually touch the network. Instead it
|
@@ -212,16 +213,16 @@ module Sidekiq
|
|
212
213
|
#
|
213
214
|
# require 'sidekiq/testing'
|
214
215
|
#
|
215
|
-
# assert_equal 0,
|
216
|
-
#
|
217
|
-
# assert_equal 1,
|
218
|
-
# assert_equal :something,
|
216
|
+
# assert_equal 0, HardJob.jobs.size
|
217
|
+
# HardJob.perform_async(:something)
|
218
|
+
# assert_equal 1, HardJob.jobs.size
|
219
|
+
# assert_equal :something, HardJob.jobs[0]['args'][0]
|
219
220
|
#
|
220
221
|
# assert_equal 0, Sidekiq::Extensions::DelayedMailer.jobs.size
|
221
222
|
# MyMailer.delay.send_welcome_email('foo@example.com')
|
222
223
|
# assert_equal 1, Sidekiq::Extensions::DelayedMailer.jobs.size
|
223
224
|
#
|
224
|
-
# You can also clear and drain all
|
225
|
+
# You can also clear and drain all job types:
|
225
226
|
#
|
226
227
|
# assert_equal 0, Sidekiq::Extensions::DelayedMailer.jobs.size
|
227
228
|
# assert_equal 0, Sidekiq::Extensions::DelayedModel.jobs.size
|
@@ -241,14 +242,14 @@ module Sidekiq
|
|
241
242
|
#
|
242
243
|
# RSpec.configure do |config|
|
243
244
|
# config.before(:each) do
|
244
|
-
# Sidekiq::
|
245
|
+
# Sidekiq::Job.clear_all
|
245
246
|
# end
|
246
247
|
# end
|
247
248
|
#
|
248
249
|
# or for acceptance testing, i.e. with cucumber:
|
249
250
|
#
|
250
251
|
# AfterStep do
|
251
|
-
# Sidekiq::
|
252
|
+
# Sidekiq::Job.drain_all
|
252
253
|
# end
|
253
254
|
#
|
254
255
|
# When I sign up as "foo@example.com"
|
@@ -262,7 +263,7 @@ module Sidekiq
|
|
262
263
|
|
263
264
|
# Jobs queued for this worker
|
264
265
|
def jobs
|
265
|
-
Queues.
|
266
|
+
Queues.jobs_by_class[to_s]
|
266
267
|
end
|
267
268
|
|
268
269
|
# Clear all jobs for this worker
|
@@ -288,11 +289,11 @@ module Sidekiq
|
|
288
289
|
end
|
289
290
|
|
290
291
|
def process_job(job)
|
291
|
-
|
292
|
-
|
293
|
-
|
294
|
-
Sidekiq::Testing.server_middleware.invoke(
|
295
|
-
execute_job(
|
292
|
+
inst = new
|
293
|
+
inst.jid = job["jid"]
|
294
|
+
inst.bid = job["bid"] if inst.respond_to?(:bid=)
|
295
|
+
Sidekiq::Testing.server_middleware.invoke(inst, job, job["queue"]) do
|
296
|
+
execute_job(inst, job["args"])
|
296
297
|
end
|
297
298
|
end
|
298
299
|
|
@@ -306,18 +307,18 @@ module Sidekiq
|
|
306
307
|
Queues.jobs_by_queue.values.flatten
|
307
308
|
end
|
308
309
|
|
309
|
-
# Clear all queued jobs
|
310
|
+
# Clear all queued jobs
|
310
311
|
def clear_all
|
311
312
|
Queues.clear_all
|
312
313
|
end
|
313
314
|
|
314
|
-
# Drain all queued jobs
|
315
|
+
# Drain (execute) all queued jobs
|
315
316
|
def drain_all
|
316
317
|
while jobs.any?
|
317
|
-
|
318
|
+
job_classes = jobs.map { |job| job["class"] }.uniq
|
318
319
|
|
319
|
-
|
320
|
-
Sidekiq::Testing.constantize(
|
320
|
+
job_classes.each do |job_class|
|
321
|
+
Sidekiq::Testing.constantize(job_class).drain
|
321
322
|
end
|
322
323
|
end
|
323
324
|
end
|
@@ -338,7 +339,5 @@ module Sidekiq
|
|
338
339
|
end
|
339
340
|
|
340
341
|
if defined?(::Rails) && Rails.respond_to?(:env) && !Rails.env.test? && !$TESTING
|
341
|
-
|
342
|
-
puts("⛔️ WARNING: Sidekiq testing API enabled, but this is not the test environment. Your jobs will not go to Redis.")
|
343
|
-
puts("**************************************************")
|
342
|
+
warn("⛔️ WARNING: Sidekiq testing API enabled, but this is not the test environment. Your jobs will not go to Redis.", uplevel: 1)
|
344
343
|
end
|
@@ -0,0 +1,45 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require "securerandom"
|
4
|
+
require "sidekiq/client"
|
5
|
+
|
6
|
+
module Sidekiq
|
7
|
+
class TransactionAwareClient
|
8
|
+
def initialize(redis_pool)
|
9
|
+
@redis_client = Client.new(redis_pool)
|
10
|
+
end
|
11
|
+
|
12
|
+
def push(item)
|
13
|
+
# pre-allocate the JID so we can return it immediately and
|
14
|
+
# save it to the database as part of the transaction.
|
15
|
+
item["jid"] ||= SecureRandom.hex(12)
|
16
|
+
AfterCommitEverywhere.after_commit { @redis_client.push(item) }
|
17
|
+
item["jid"]
|
18
|
+
end
|
19
|
+
|
20
|
+
##
|
21
|
+
# We don't provide transactionality for push_bulk because we don't want
|
22
|
+
# to hold potentially hundreds of thousands of job records in memory due to
|
23
|
+
# a long running enqueue process.
|
24
|
+
def push_bulk(items)
|
25
|
+
@redis_client.push_bulk(items)
|
26
|
+
end
|
27
|
+
end
|
28
|
+
end
|
29
|
+
|
30
|
+
##
|
31
|
+
# Use `Sidekiq.transactional_push!` in your sidekiq.rb initializer
|
32
|
+
module Sidekiq
|
33
|
+
def self.transactional_push!
|
34
|
+
begin
|
35
|
+
require "after_commit_everywhere"
|
36
|
+
rescue LoadError
|
37
|
+
Sidekiq.logger.error("You need to add after_commit_everywhere to your Gemfile to use Sidekiq's transactional client")
|
38
|
+
raise
|
39
|
+
end
|
40
|
+
|
41
|
+
default_job_options["client_class"] = Sidekiq::TransactionAwareClient
|
42
|
+
Sidekiq::JobUtil::TRANSIENT_ATTRIBUTES << "client_class"
|
43
|
+
true
|
44
|
+
end
|
45
|
+
end
|
data/lib/sidekiq/version.rb
CHANGED
data/lib/sidekiq/web/action.rb
CHANGED
@@ -68,7 +68,7 @@ module Sidekiq
|
|
68
68
|
end
|
69
69
|
|
70
70
|
def json(payload)
|
71
|
-
[200, {"Content-Type" => "application/json", "Cache-Control" => "no-
|
71
|
+
[200, {"Content-Type" => "application/json", "Cache-Control" => "private, no-store"}, [Sidekiq.dump_json(payload)]]
|
72
72
|
end
|
73
73
|
|
74
74
|
def initialize(env, block)
|
@@ -50,7 +50,10 @@ module Sidekiq
|
|
50
50
|
|
51
51
|
get "/" do
|
52
52
|
@redis_info = redis_info.select { |k, v| REDIS_KEYS.include? k }
|
53
|
-
|
53
|
+
days = (params["days"] || 30).to_i
|
54
|
+
return halt(401) if days < 1 || days > 180
|
55
|
+
|
56
|
+
stats_history = Sidekiq::Stats::History.new(days)
|
54
57
|
@processed_history = stats_history.processed
|
55
58
|
@failed_history = stats_history.failed
|
56
59
|
|
@@ -91,8 +94,8 @@ module Sidekiq
|
|
91
94
|
|
92
95
|
@count = (params["count"] || 25).to_i
|
93
96
|
@queue = Sidekiq::Queue.new(@name)
|
94
|
-
(@current_page, @total_size, @
|
95
|
-
@
|
97
|
+
(@current_page, @total_size, @jobs) = page("queue:#{@name}", params["page"], @count, reverse: params["direction"] == "asc")
|
98
|
+
@jobs = @jobs.map { |msg| Sidekiq::JobRecord.new(msg, @name) }
|
96
99
|
|
97
100
|
erb(:queue)
|
98
101
|
end
|
@@ -113,7 +116,7 @@ module Sidekiq
|
|
113
116
|
|
114
117
|
post "/queues/:name/delete" do
|
115
118
|
name = route_params[:name]
|
116
|
-
Sidekiq::
|
119
|
+
Sidekiq::JobRecord.new(params["key_val"], name).delete
|
117
120
|
|
118
121
|
redirect_with_query("#{root_path}queues/#{CGI.escape(name)}")
|
119
122
|
end
|
@@ -299,7 +302,7 @@ module Sidekiq
|
|
299
302
|
return [404, {"Content-Type" => "text/plain", "X-Cascade" => "pass"}, ["Not Found"]] unless action
|
300
303
|
|
301
304
|
app = @klass
|
302
|
-
resp = catch(:halt) do
|
305
|
+
resp = catch(:halt) do
|
303
306
|
self.class.run_befores(app, action)
|
304
307
|
action.instance_exec env, &action.block
|
305
308
|
ensure
|
@@ -314,7 +317,7 @@ module Sidekiq
|
|
314
317
|
# rendered content goes here
|
315
318
|
headers = {
|
316
319
|
"Content-Type" => "text/html",
|
317
|
-
"Cache-Control" => "no-
|
320
|
+
"Cache-Control" => "private, no-store",
|
318
321
|
"Content-Language" => action.locale,
|
319
322
|
"Content-Security-Policy" => CSP_HEADER
|
320
323
|
}
|
@@ -143,7 +143,7 @@ module Sidekiq
|
|
143
143
|
one_time_pad = SecureRandom.random_bytes(token.length)
|
144
144
|
encrypted_token = xor_byte_strings(one_time_pad, token)
|
145
145
|
masked_token = one_time_pad + encrypted_token
|
146
|
-
Base64.
|
146
|
+
Base64.urlsafe_encode64(masked_token)
|
147
147
|
end
|
148
148
|
|
149
149
|
# Essentially the inverse of +mask_token+.
|
@@ -169,7 +169,7 @@ module Sidekiq
|
|
169
169
|
end
|
170
170
|
|
171
171
|
def decode_token(token)
|
172
|
-
Base64.
|
172
|
+
Base64.urlsafe_decode64(token)
|
173
173
|
end
|
174
174
|
|
175
175
|
def xor_byte_strings(s1, s2)
|
data/lib/sidekiq/web/helpers.rb
CHANGED
@@ -10,14 +10,13 @@ module Sidekiq
|
|
10
10
|
module WebHelpers
|
11
11
|
def strings(lang)
|
12
12
|
@strings ||= {}
|
13
|
-
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
end
|
13
|
+
|
14
|
+
# Allow sidekiq-web extensions to add locale paths
|
15
|
+
# so extensions can be localized
|
16
|
+
@strings[lang] ||= settings.locales.each_with_object({}) do |path, global|
|
17
|
+
find_locale_files(lang).each do |file|
|
18
|
+
strs = YAML.load(File.open(file))
|
19
|
+
global.merge!(strs[lang])
|
21
20
|
end
|
22
21
|
end
|
23
22
|
end
|
@@ -71,17 +70,6 @@ module Sidekiq
|
|
71
70
|
@head_html.join if defined?(@head_html)
|
72
71
|
end
|
73
72
|
|
74
|
-
def poll_path
|
75
|
-
if current_path != "" && params["poll"]
|
76
|
-
path = root_path + current_path
|
77
|
-
query_string = to_query_string(params.slice(*params.keys - %w[page poll]))
|
78
|
-
path += "?#{query_string}" unless query_string.empty?
|
79
|
-
path
|
80
|
-
else
|
81
|
-
""
|
82
|
-
end
|
83
|
-
end
|
84
|
-
|
85
73
|
def text_direction
|
86
74
|
get_locale["TextDirection"] || "ltr"
|
87
75
|
end
|
@@ -126,7 +114,7 @@ module Sidekiq
|
|
126
114
|
# within is used by Sidekiq Pro
|
127
115
|
def display_tags(job, within = nil)
|
128
116
|
job.tags.map { |tag|
|
129
|
-
"<span class='
|
117
|
+
"<span class='label label-info jobtag'>#{::Rack::Utils.escape_html(tag)}</span>"
|
130
118
|
}.join(" ")
|
131
119
|
end
|
132
120
|
|
@@ -152,8 +140,8 @@ module Sidekiq
|
|
152
140
|
params[:direction] == "asc" ? "↑" : "↓"
|
153
141
|
end
|
154
142
|
|
155
|
-
def
|
156
|
-
@
|
143
|
+
def workset
|
144
|
+
@work ||= Sidekiq::WorkSet.new
|
157
145
|
end
|
158
146
|
|
159
147
|
def processes
|
@@ -187,7 +175,7 @@ module Sidekiq
|
|
187
175
|
end
|
188
176
|
|
189
177
|
def current_status
|
190
|
-
|
178
|
+
workset.size == 0 ? "idle" : "active"
|
191
179
|
end
|
192
180
|
|
193
181
|
def relative_time(time)
|
@@ -204,7 +192,7 @@ module Sidekiq
|
|
204
192
|
[score.to_f, jid]
|
205
193
|
end
|
206
194
|
|
207
|
-
SAFE_QPARAMS = %w[page
|
195
|
+
SAFE_QPARAMS = %w[page direction]
|
208
196
|
|
209
197
|
# Merge options with current params, filter safe params, and stringify to query string
|
210
198
|
def qparams(options)
|
@@ -254,7 +242,7 @@ module Sidekiq
|
|
254
242
|
queue class args retry_count retried_at failed_at
|
255
243
|
jid error_message error_class backtrace
|
256
244
|
error_backtrace enqueued_at retry wrapped
|
257
|
-
created_at tags
|
245
|
+
created_at tags display_class
|
258
246
|
])
|
259
247
|
|
260
248
|
def retry_extra_items(retry_job)
|
@@ -313,7 +301,7 @@ module Sidekiq
|
|
313
301
|
end
|
314
302
|
|
315
303
|
def environment_title_prefix
|
316
|
-
environment = Sidekiq
|
304
|
+
environment = Sidekiq[:environment] || ENV["APP_ENV"] || ENV["RAILS_ENV"] || ENV["RACK_ENV"] || "development"
|
317
305
|
|
318
306
|
"[#{environment.upcase}] " unless environment == "production"
|
319
307
|
end
|
data/lib/sidekiq/web.rb
CHANGED
@@ -143,13 +143,14 @@ module Sidekiq
|
|
143
143
|
klass = self.class
|
144
144
|
m = middlewares
|
145
145
|
|
146
|
+
rules = []
|
147
|
+
rules = [[:all, {"Cache-Control" => "public, max-age=86400"}]] unless ENV["SIDEKIQ_WEB_TESTING"]
|
148
|
+
|
146
149
|
::Rack::Builder.new do
|
147
150
|
use Rack::Static, urls: ["/stylesheets", "/images", "/javascripts"],
|
148
|
-
|
149
|
-
|
150
|
-
|
151
|
-
[:all, {"Cache-Control" => "public, max-age=86400"}]
|
152
|
-
]
|
151
|
+
root: ASSETS,
|
152
|
+
cascade: true,
|
153
|
+
header_rules: rules
|
153
154
|
m.each { |middleware, block| use(*middleware, &block) }
|
154
155
|
use Sidekiq::Web::CsrfProtection unless $TESTING
|
155
156
|
run WebApplication.new(klass)
|
data/lib/sidekiq/worker.rb
CHANGED
@@ -9,6 +9,7 @@ module Sidekiq
|
|
9
9
|
#
|
10
10
|
# class HardWorker
|
11
11
|
# include Sidekiq::Worker
|
12
|
+
# sidekiq_options queue: 'critical', retry: 5
|
12
13
|
#
|
13
14
|
# def perform(*args)
|
14
15
|
# # do some work
|
@@ -20,6 +21,26 @@ module Sidekiq
|
|
20
21
|
# HardWorker.perform_async(1, 2, 3)
|
21
22
|
#
|
22
23
|
# Note that perform_async is a class method, perform is an instance method.
|
24
|
+
#
|
25
|
+
# Sidekiq::Worker also includes several APIs to provide compatibility with
|
26
|
+
# ActiveJob.
|
27
|
+
#
|
28
|
+
# class SomeWorker
|
29
|
+
# include Sidekiq::Worker
|
30
|
+
# queue_as :critical
|
31
|
+
#
|
32
|
+
# def perform(...)
|
33
|
+
# end
|
34
|
+
# end
|
35
|
+
#
|
36
|
+
# SomeWorker.set(wait_until: 1.hour).perform_async(123)
|
37
|
+
#
|
38
|
+
# Note that arguments passed to the job must still obey Sidekiq's
|
39
|
+
# best practice for simple, JSON-native data types. Sidekiq will not
|
40
|
+
# implement ActiveJob's more complex argument serialization. For
|
41
|
+
# this reason, we don't implement `perform_later` as our call semantics
|
42
|
+
# are very different.
|
43
|
+
#
|
23
44
|
module Worker
|
24
45
|
##
|
25
46
|
# The Options module is extracted so we can include it in ActiveJob::Base
|
@@ -61,7 +82,7 @@ module Sidekiq
|
|
61
82
|
end
|
62
83
|
|
63
84
|
def get_sidekiq_options # :nodoc:
|
64
|
-
self.sidekiq_options_hash ||= Sidekiq.
|
85
|
+
self.sidekiq_options_hash ||= Sidekiq.default_job_options
|
65
86
|
end
|
66
87
|
|
67
88
|
def sidekiq_class_attribute(*attrs)
|
@@ -150,33 +171,97 @@ module Sidekiq
|
|
150
171
|
# SomeWorker.set(queue: 'foo').perform_async(....)
|
151
172
|
#
|
152
173
|
class Setter
|
174
|
+
include Sidekiq::JobUtil
|
175
|
+
|
153
176
|
def initialize(klass, opts)
|
154
177
|
@klass = klass
|
155
|
-
|
178
|
+
# NB: the internal hash always has stringified keys
|
179
|
+
@opts = opts.transform_keys(&:to_s)
|
180
|
+
|
181
|
+
# ActiveJob compatibility
|
182
|
+
interval = @opts.delete("wait_until") || @opts.delete("wait")
|
183
|
+
at(interval) if interval
|
156
184
|
end
|
157
185
|
|
158
186
|
def set(options)
|
159
|
-
|
187
|
+
hash = options.transform_keys(&:to_s)
|
188
|
+
interval = hash.delete("wait_until") || @opts.delete("wait")
|
189
|
+
@opts.merge!(hash)
|
190
|
+
at(interval) if interval
|
160
191
|
self
|
161
192
|
end
|
162
193
|
|
163
194
|
def perform_async(*args)
|
164
|
-
@
|
195
|
+
if @opts["sync"] == true
|
196
|
+
perform_inline(*args)
|
197
|
+
else
|
198
|
+
@klass.client_push(@opts.merge("args" => args, "class" => @klass))
|
199
|
+
end
|
200
|
+
end
|
201
|
+
|
202
|
+
# Explicit inline execution of a job. Returns nil if the job did not
|
203
|
+
# execute, true otherwise.
|
204
|
+
def perform_inline(*args)
|
205
|
+
raw = @opts.merge("args" => args, "class" => @klass)
|
206
|
+
|
207
|
+
# validate and normalize payload
|
208
|
+
item = normalize_item(raw)
|
209
|
+
queue = item["queue"]
|
210
|
+
|
211
|
+
# run client-side middleware
|
212
|
+
result = Sidekiq.client_middleware.invoke(item["class"], item, queue, Sidekiq.redis_pool) do
|
213
|
+
item
|
214
|
+
end
|
215
|
+
return nil unless result
|
216
|
+
|
217
|
+
# round-trip the payload via JSON
|
218
|
+
msg = Sidekiq.load_json(Sidekiq.dump_json(item))
|
219
|
+
|
220
|
+
# prepare the job instance
|
221
|
+
klass = msg["class"].constantize
|
222
|
+
job = klass.new
|
223
|
+
job.jid = msg["jid"]
|
224
|
+
job.bid = msg["bid"] if job.respond_to?(:bid)
|
225
|
+
|
226
|
+
# run the job through server-side middleware
|
227
|
+
result = Sidekiq.server_middleware.invoke(job, msg, msg["queue"]) do
|
228
|
+
# perform it
|
229
|
+
job.perform(*msg["args"])
|
230
|
+
true
|
231
|
+
end
|
232
|
+
return nil unless result
|
233
|
+
# jobs do not return a result. they should store any
|
234
|
+
# modified state.
|
235
|
+
true
|
236
|
+
end
|
237
|
+
alias_method :perform_sync, :perform_inline
|
238
|
+
|
239
|
+
def perform_bulk(args, batch_size: 1_000)
|
240
|
+
client = @klass.build_client
|
241
|
+
result = args.each_slice(batch_size).flat_map do |slice|
|
242
|
+
client.push_bulk(@opts.merge("class" => @klass, "args" => slice))
|
243
|
+
end
|
244
|
+
|
245
|
+
result.is_a?(Enumerator::Lazy) ? result.force : result
|
165
246
|
end
|
166
247
|
|
167
248
|
# +interval+ must be a timestamp, numeric or something that acts
|
168
249
|
# numeric (like an activesupport time interval).
|
169
250
|
def perform_in(interval, *args)
|
251
|
+
at(interval).perform_async(*args)
|
252
|
+
end
|
253
|
+
alias_method :perform_at, :perform_in
|
254
|
+
|
255
|
+
private
|
256
|
+
|
257
|
+
def at(interval)
|
170
258
|
int = interval.to_f
|
171
259
|
now = Time.now.to_f
|
172
260
|
ts = (int < 1_000_000_000 ? now + int : int)
|
173
|
-
|
174
|
-
payload = @opts.merge("class" => @klass, "args" => args)
|
175
261
|
# Optimization to enqueue something now that is scheduled to go out now or in the past
|
176
|
-
|
177
|
-
|
262
|
+
@opts["at"] = ts if ts > now
|
263
|
+
self
|
178
264
|
end
|
179
|
-
alias_method :perform_at, :perform_in
|
180
265
|
end
|
181
266
|
|
182
267
|
module ClassMethods
|
@@ -192,12 +277,46 @@ module Sidekiq
|
|
192
277
|
raise ArgumentError, "Do not call .delay_until on a Sidekiq::Worker class, call .perform_at"
|
193
278
|
end
|
194
279
|
|
280
|
+
def queue_as(q)
|
281
|
+
sidekiq_options("queue" => q.to_s)
|
282
|
+
end
|
283
|
+
|
195
284
|
def set(options)
|
196
285
|
Setter.new(self, options)
|
197
286
|
end
|
198
287
|
|
199
288
|
def perform_async(*args)
|
200
|
-
|
289
|
+
Setter.new(self, {}).perform_async(*args)
|
290
|
+
end
|
291
|
+
|
292
|
+
# Inline execution of job's perform method after passing through Sidekiq.client_middleware and Sidekiq.server_middleware
|
293
|
+
def perform_inline(*args)
|
294
|
+
Setter.new(self, {}).perform_inline(*args)
|
295
|
+
end
|
296
|
+
alias_method :perform_sync, :perform_inline
|
297
|
+
|
298
|
+
##
|
299
|
+
# Push a large number of jobs to Redis, while limiting the batch of
|
300
|
+
# each job payload to 1,000. This method helps cut down on the number
|
301
|
+
# of round trips to Redis, which can increase the performance of enqueueing
|
302
|
+
# large numbers of jobs.
|
303
|
+
#
|
304
|
+
# +items+ must be an Array of Arrays.
|
305
|
+
#
|
306
|
+
# For finer-grained control, use `Sidekiq::Client.push_bulk` directly.
|
307
|
+
#
|
308
|
+
# Example (3 Redis round trips):
|
309
|
+
#
|
310
|
+
# SomeWorker.perform_async(1)
|
311
|
+
# SomeWorker.perform_async(2)
|
312
|
+
# SomeWorker.perform_async(3)
|
313
|
+
#
|
314
|
+
# Would instead become (1 Redis round trip):
|
315
|
+
#
|
316
|
+
# SomeWorker.perform_bulk([[1], [2], [3]])
|
317
|
+
#
|
318
|
+
def perform_bulk(*args, **kwargs)
|
319
|
+
Setter.new(self, {}).perform_bulk(*args, **kwargs)
|
201
320
|
end
|
202
321
|
|
203
322
|
# +interval+ must be a timestamp, numeric or something that acts
|
@@ -234,10 +353,14 @@ module Sidekiq
|
|
234
353
|
end
|
235
354
|
|
236
355
|
def client_push(item) # :nodoc:
|
237
|
-
|
238
|
-
|
356
|
+
raise ArgumentError, "Job payloads should contain no Symbols: #{item}" if item.any? { |k, v| k.is_a?(::Symbol) }
|
357
|
+
build_client.push(item)
|
358
|
+
end
|
239
359
|
|
240
|
-
|
360
|
+
def build_client # :nodoc:
|
361
|
+
pool = Thread.current[:sidekiq_via_pool] || get_sidekiq_options["pool"] || Sidekiq.redis_pool
|
362
|
+
client_class = get_sidekiq_options["client_class"] || Sidekiq::Client
|
363
|
+
client_class.new(pool)
|
241
364
|
end
|
242
365
|
end
|
243
366
|
end
|