sidekiq 5.2.9 → 6.4.1
Sign up to get free protection for your applications and to get access to all the features.
Potentially problematic release.
This version of sidekiq might be problematic. Click here for more details.
- checksums.yaml +4 -4
- data/Changes.md +318 -1
- data/LICENSE +3 -3
- data/README.md +23 -34
- data/bin/sidekiq +27 -3
- data/bin/sidekiqload +67 -61
- data/bin/sidekiqmon +8 -0
- data/lib/generators/sidekiq/job_generator.rb +57 -0
- data/lib/generators/sidekiq/templates/{worker.rb.erb → job.rb.erb} +2 -2
- data/lib/generators/sidekiq/templates/{worker_spec.rb.erb → job_spec.rb.erb} +1 -1
- data/lib/generators/sidekiq/templates/{worker_test.rb.erb → job_test.rb.erb} +1 -1
- data/lib/sidekiq/api.rb +335 -267
- data/lib/sidekiq/cli.rb +164 -182
- data/lib/sidekiq/client.rb +58 -61
- data/lib/sidekiq/delay.rb +7 -6
- data/lib/sidekiq/exception_handler.rb +10 -12
- data/lib/sidekiq/extensions/action_mailer.rb +13 -22
- data/lib/sidekiq/extensions/active_record.rb +13 -10
- data/lib/sidekiq/extensions/class_methods.rb +14 -11
- data/lib/sidekiq/extensions/generic_proxy.rb +6 -4
- data/lib/sidekiq/fetch.rb +40 -32
- data/lib/sidekiq/job.rb +13 -0
- data/lib/sidekiq/job_logger.rb +33 -7
- data/lib/sidekiq/job_retry.rb +70 -71
- data/lib/sidekiq/job_util.rb +65 -0
- data/lib/sidekiq/launcher.rb +161 -71
- data/lib/sidekiq/logger.rb +170 -0
- data/lib/sidekiq/manager.rb +17 -21
- data/lib/sidekiq/middleware/chain.rb +20 -8
- data/lib/sidekiq/middleware/current_attributes.rb +57 -0
- data/lib/sidekiq/middleware/i18n.rb +5 -7
- data/lib/sidekiq/monitor.rb +133 -0
- data/lib/sidekiq/paginator.rb +20 -16
- data/lib/sidekiq/processor.rb +71 -70
- data/lib/sidekiq/rails.rb +40 -37
- data/lib/sidekiq/redis_connection.rb +48 -48
- data/lib/sidekiq/scheduled.rb +62 -28
- data/lib/sidekiq/sd_notify.rb +149 -0
- data/lib/sidekiq/systemd.rb +24 -0
- data/lib/sidekiq/testing/inline.rb +2 -1
- data/lib/sidekiq/testing.rb +36 -27
- data/lib/sidekiq/util.rb +57 -15
- data/lib/sidekiq/version.rb +2 -1
- data/lib/sidekiq/web/action.rb +15 -11
- data/lib/sidekiq/web/application.rb +88 -75
- data/lib/sidekiq/web/csrf_protection.rb +180 -0
- data/lib/sidekiq/web/helpers.rb +109 -92
- data/lib/sidekiq/web/router.rb +23 -19
- data/lib/sidekiq/web.rb +61 -105
- data/lib/sidekiq/worker.rb +247 -105
- data/lib/sidekiq.rb +77 -44
- data/sidekiq.gemspec +23 -16
- data/web/assets/images/apple-touch-icon.png +0 -0
- data/web/assets/javascripts/application.js +83 -64
- data/web/assets/javascripts/dashboard.js +54 -73
- data/web/assets/stylesheets/application-dark.css +143 -0
- data/web/assets/stylesheets/application-rtl.css +0 -4
- data/web/assets/stylesheets/application.css +45 -232
- data/web/locales/ar.yml +8 -2
- data/web/locales/de.yml +14 -2
- data/web/locales/en.yml +6 -1
- data/web/locales/es.yml +18 -2
- data/web/locales/fr.yml +10 -3
- data/web/locales/ja.yml +7 -1
- data/web/locales/lt.yml +83 -0
- data/web/locales/pl.yml +4 -4
- data/web/locales/ru.yml +4 -0
- data/web/locales/vi.yml +83 -0
- data/web/views/_footer.erb +1 -1
- data/web/views/_job_info.erb +3 -2
- data/web/views/_poll_link.erb +2 -5
- data/web/views/_summary.erb +7 -7
- data/web/views/busy.erb +54 -20
- data/web/views/dashboard.erb +22 -14
- data/web/views/dead.erb +3 -3
- data/web/views/layout.erb +3 -1
- data/web/views/morgue.erb +9 -6
- data/web/views/queue.erb +19 -10
- data/web/views/queues.erb +10 -2
- data/web/views/retries.erb +11 -8
- data/web/views/retry.erb +3 -3
- data/web/views/scheduled.erb +5 -2
- metadata +34 -64
- data/.circleci/config.yml +0 -61
- data/.github/contributing.md +0 -32
- data/.github/issue_template.md +0 -11
- data/.gitignore +0 -15
- data/.travis.yml +0 -11
- data/3.0-Upgrade.md +0 -70
- data/4.0-Upgrade.md +0 -53
- data/5.0-Upgrade.md +0 -56
- data/COMM-LICENSE +0 -97
- data/Ent-Changes.md +0 -238
- data/Gemfile +0 -23
- data/Pro-2.0-Upgrade.md +0 -138
- data/Pro-3.0-Upgrade.md +0 -44
- data/Pro-4.0-Upgrade.md +0 -35
- data/Pro-Changes.md +0 -759
- data/Rakefile +0 -9
- data/bin/sidekiqctl +0 -20
- data/code_of_conduct.md +0 -50
- data/lib/generators/sidekiq/worker_generator.rb +0 -49
- data/lib/sidekiq/core_ext.rb +0 -1
- data/lib/sidekiq/ctl.rb +0 -221
- data/lib/sidekiq/logging.rb +0 -122
- data/lib/sidekiq/middleware/server/active_record.rb +0 -23
data/lib/sidekiq/client.rb
CHANGED
@@ -1,9 +1,12 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
|
-
|
3
|
-
require
|
2
|
+
|
3
|
+
require "securerandom"
|
4
|
+
require "sidekiq/middleware/chain"
|
5
|
+
require "sidekiq/job_util"
|
4
6
|
|
5
7
|
module Sidekiq
|
6
8
|
class Client
|
9
|
+
include Sidekiq::JobUtil
|
7
10
|
|
8
11
|
##
|
9
12
|
# Define client-side middleware:
|
@@ -19,7 +22,7 @@ module Sidekiq
|
|
19
22
|
#
|
20
23
|
def middleware(&block)
|
21
24
|
@chain ||= Sidekiq.client_middleware
|
22
|
-
if
|
25
|
+
if block
|
23
26
|
@chain = @chain.dup
|
24
27
|
yield @chain
|
25
28
|
end
|
@@ -38,7 +41,7 @@ module Sidekiq
|
|
38
41
|
# Generally this is only needed for very large Sidekiq installs processing
|
39
42
|
# thousands of jobs per second. I don't recommend sharding unless you
|
40
43
|
# cannot scale any other way (e.g. splitting your app into smaller apps).
|
41
|
-
def initialize(redis_pool=nil)
|
44
|
+
def initialize(redis_pool = nil)
|
42
45
|
@redis_pool = redis_pool || Thread.current[:sidekiq_via_pool] || Sidekiq.redis_pool
|
43
46
|
end
|
44
47
|
|
@@ -68,11 +71,11 @@ module Sidekiq
|
|
68
71
|
#
|
69
72
|
def push(item)
|
70
73
|
normed = normalize_item(item)
|
71
|
-
payload = process_single(item[
|
74
|
+
payload = process_single(item["class"], normed)
|
72
75
|
|
73
76
|
if payload
|
74
77
|
raw_push([payload])
|
75
|
-
payload[
|
78
|
+
payload["jid"]
|
76
79
|
end
|
77
80
|
end
|
78
81
|
|
@@ -90,19 +93,25 @@ module Sidekiq
|
|
90
93
|
# Returns an array of the of pushed jobs' jids. The number of jobs pushed can be less
|
91
94
|
# than the number given if the middleware stopped processing for one or more jobs.
|
92
95
|
def push_bulk(items)
|
93
|
-
|
94
|
-
|
95
|
-
|
96
|
+
args = items["args"]
|
97
|
+
raise ArgumentError, "Bulk arguments must be an Array of Arrays: [[1], [2]]" unless args.is_a?(Array) && args.all?(Array)
|
98
|
+
return [] if args.empty? # no jobs to push
|
99
|
+
|
100
|
+
at = items.delete("at")
|
101
|
+
raise ArgumentError, "Job 'at' must be a Numeric or an Array of Numeric timestamps" if at && (Array(at).empty? || !Array(at).all? { |entry| entry.is_a?(Numeric) })
|
102
|
+
raise ArgumentError, "Job 'at' Array must have same size as 'args' Array" if at.is_a?(Array) && at.size != args.size
|
96
103
|
|
97
104
|
normed = normalize_item(items)
|
98
|
-
payloads =
|
99
|
-
copy = normed.merge(
|
100
|
-
|
101
|
-
|
102
|
-
|
103
|
-
|
104
|
-
|
105
|
-
|
105
|
+
payloads = args.map.with_index { |job_args, index|
|
106
|
+
copy = normed.merge("args" => job_args, "jid" => SecureRandom.hex(12))
|
107
|
+
copy["at"] = (at.is_a?(Array) ? at[index] : at) if at
|
108
|
+
|
109
|
+
result = process_single(items["class"], copy)
|
110
|
+
result || nil
|
111
|
+
}.compact
|
112
|
+
|
113
|
+
raw_push(payloads) unless payloads.empty?
|
114
|
+
payloads.collect { |payload| payload["jid"] }
|
106
115
|
end
|
107
116
|
|
108
117
|
# Allows sharding of jobs across any number of Redis instances. All jobs
|
@@ -127,7 +136,6 @@ module Sidekiq
|
|
127
136
|
end
|
128
137
|
|
129
138
|
class << self
|
130
|
-
|
131
139
|
def push(item)
|
132
140
|
new.push(item)
|
133
141
|
end
|
@@ -145,14 +153,14 @@ module Sidekiq
|
|
145
153
|
# Messages are enqueued to the 'default' queue.
|
146
154
|
#
|
147
155
|
def enqueue(klass, *args)
|
148
|
-
klass.client_push(
|
156
|
+
klass.client_push("class" => klass, "args" => args)
|
149
157
|
end
|
150
158
|
|
151
159
|
# Example usage:
|
152
160
|
# Sidekiq::Client.enqueue_to(:queue_name, MyWorker, 'foo', 1, :bat => 'bar')
|
153
161
|
#
|
154
162
|
def enqueue_to(queue, klass, *args)
|
155
|
-
klass.client_push(
|
163
|
+
klass.client_push("queue" => queue, "class" => klass, "args" => args)
|
156
164
|
end
|
157
165
|
|
158
166
|
# Example usage:
|
@@ -163,8 +171,8 @@ module Sidekiq
|
|
163
171
|
now = Time.now.to_f
|
164
172
|
ts = (int < 1_000_000_000 ? now + int : int)
|
165
173
|
|
166
|
-
item = {
|
167
|
-
item.delete(
|
174
|
+
item = {"class" => klass, "args" => args, "at" => ts, "queue" => queue}
|
175
|
+
item.delete("at") if ts <= now
|
168
176
|
|
169
177
|
klass.client_push(item)
|
170
178
|
end
|
@@ -181,63 +189,52 @@ module Sidekiq
|
|
181
189
|
|
182
190
|
def raw_push(payloads)
|
183
191
|
@redis_pool.with do |conn|
|
184
|
-
|
185
|
-
|
192
|
+
retryable = true
|
193
|
+
begin
|
194
|
+
conn.pipelined do |pipeline|
|
195
|
+
atomic_push(pipeline, payloads)
|
196
|
+
end
|
197
|
+
rescue Redis::BaseError => ex
|
198
|
+
# 2550 Failover can cause the server to become a replica, need
|
199
|
+
# to disconnect and reopen the socket to get back to the primary.
|
200
|
+
# 4495 Use the same logic if we have a "Not enough replicas" error from the primary
|
201
|
+
# 4985 Use the same logic when a blocking command is force-unblocked
|
202
|
+
# The retry logic is copied from sidekiq.rb
|
203
|
+
if retryable && ex.message =~ /READONLY|NOREPLICAS|UNBLOCKED/
|
204
|
+
conn.disconnect!
|
205
|
+
retryable = false
|
206
|
+
retry
|
207
|
+
end
|
208
|
+
raise
|
186
209
|
end
|
187
210
|
end
|
188
211
|
true
|
189
212
|
end
|
190
213
|
|
191
214
|
def atomic_push(conn, payloads)
|
192
|
-
if payloads.first
|
193
|
-
conn.zadd(
|
194
|
-
at = hash.delete(
|
215
|
+
if payloads.first.key?("at")
|
216
|
+
conn.zadd("schedule", payloads.map { |hash|
|
217
|
+
at = hash.delete("at").to_s
|
195
218
|
[at, Sidekiq.dump_json(hash)]
|
196
|
-
|
219
|
+
})
|
197
220
|
else
|
198
|
-
|
221
|
+
queue = payloads.first["queue"]
|
199
222
|
now = Time.now.to_f
|
200
|
-
to_push = payloads.map
|
201
|
-
entry[
|
223
|
+
to_push = payloads.map { |entry|
|
224
|
+
entry["enqueued_at"] = now
|
202
225
|
Sidekiq.dump_json(entry)
|
203
|
-
|
204
|
-
conn.sadd(
|
205
|
-
conn.lpush("queue:#{
|
226
|
+
}
|
227
|
+
conn.sadd("queues", queue)
|
228
|
+
conn.lpush("queue:#{queue}", to_push)
|
206
229
|
end
|
207
230
|
end
|
208
231
|
|
209
232
|
def process_single(worker_class, item)
|
210
|
-
queue = item[
|
233
|
+
queue = item["queue"]
|
211
234
|
|
212
235
|
middleware.invoke(worker_class, item, queue, @redis_pool) do
|
213
236
|
item
|
214
237
|
end
|
215
238
|
end
|
216
|
-
|
217
|
-
def normalize_item(item)
|
218
|
-
raise(ArgumentError, "Job must be a Hash with 'class' and 'args' keys: { 'class' => SomeWorker, 'args' => ['bob', 1, :foo => 'bar'] }") unless item.is_a?(Hash) && item.has_key?('class') && item.has_key?('args')
|
219
|
-
raise(ArgumentError, "Job args must be an Array") unless item['args'].is_a?(Array)
|
220
|
-
raise(ArgumentError, "Job class must be either a Class or String representation of the class name") unless item['class'].is_a?(Class) || item['class'].is_a?(String)
|
221
|
-
raise(ArgumentError, "Job 'at' must be a Numeric timestamp") if item.has_key?('at') && !item['at'].is_a?(Numeric)
|
222
|
-
#raise(ArgumentError, "Arguments must be native JSON types, see https://github.com/mperham/sidekiq/wiki/Best-Practices") unless JSON.load(JSON.dump(item['args'])) == item['args']
|
223
|
-
|
224
|
-
normalized_hash(item['class'])
|
225
|
-
.each{ |key, value| item[key] = value if item[key].nil? }
|
226
|
-
|
227
|
-
item['class'] = item['class'].to_s
|
228
|
-
item['queue'] = item['queue'].to_s
|
229
|
-
item['jid'] ||= SecureRandom.hex(12)
|
230
|
-
item['created_at'] ||= Time.now.to_f
|
231
|
-
item
|
232
|
-
end
|
233
|
-
|
234
|
-
def normalized_hash(item_class)
|
235
|
-
if item_class.is_a?(Class)
|
236
|
-
raise(ArgumentError, "Message must include a Sidekiq::Worker class, not class name: #{item_class.ancestors.inspect}") if !item_class.respond_to?('get_sidekiq_options')
|
237
|
-
item_class.get_sidekiq_options
|
238
|
-
else
|
239
|
-
Sidekiq.default_worker_options
|
240
|
-
end
|
241
|
-
end
|
242
239
|
end
|
243
240
|
end
|
data/lib/sidekiq/delay.rb
CHANGED
@@ -1,11 +1,13 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
|
+
|
2
3
|
module Sidekiq
|
3
4
|
module Extensions
|
4
|
-
|
5
5
|
def self.enable_delay!
|
6
|
+
warn "Sidekiq's Delayed Extensions will be removed in Sidekiq 7.0", uplevel: 1
|
7
|
+
|
6
8
|
if defined?(::ActiveSupport)
|
7
|
-
require
|
8
|
-
require
|
9
|
+
require "sidekiq/extensions/active_record"
|
10
|
+
require "sidekiq/extensions/action_mailer"
|
9
11
|
|
10
12
|
# Need to patch Psych so it can autoload classes whose names are serialized
|
11
13
|
# in the delayed YAML.
|
@@ -19,7 +21,7 @@ module Sidekiq
|
|
19
21
|
end
|
20
22
|
end
|
21
23
|
|
22
|
-
require
|
24
|
+
require "sidekiq/extensions/class_methods"
|
23
25
|
Module.__send__(:include, Sidekiq::Extensions::Klass)
|
24
26
|
end
|
25
27
|
|
@@ -27,7 +29,7 @@ module Sidekiq
|
|
27
29
|
def resolve_class(klass_name)
|
28
30
|
return nil if !klass_name || klass_name.empty?
|
29
31
|
# constantize
|
30
|
-
names = klass_name.split(
|
32
|
+
names = klass_name.split("::")
|
31
33
|
names.shift if names.empty? || names.first.empty?
|
32
34
|
|
33
35
|
names.inject(Object) do |constant, name|
|
@@ -39,4 +41,3 @@ module Sidekiq
|
|
39
41
|
end
|
40
42
|
end
|
41
43
|
end
|
42
|
-
|
@@ -1,12 +1,12 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
|
-
|
2
|
+
|
3
|
+
require "sidekiq"
|
3
4
|
|
4
5
|
module Sidekiq
|
5
6
|
module ExceptionHandler
|
6
|
-
|
7
7
|
class Logger
|
8
|
-
def call(ex,
|
9
|
-
Sidekiq.logger.warn(Sidekiq.dump_json(
|
8
|
+
def call(ex, ctx)
|
9
|
+
Sidekiq.logger.warn(Sidekiq.dump_json(ctx)) unless ctx.empty?
|
10
10
|
Sidekiq.logger.warn("#{ex.class.name}: #{ex.message}")
|
11
11
|
Sidekiq.logger.warn(ex.backtrace.join("\n")) unless ex.backtrace.nil?
|
12
12
|
end
|
@@ -14,15 +14,13 @@ module Sidekiq
|
|
14
14
|
Sidekiq.error_handlers << Sidekiq::ExceptionHandler::Logger.new
|
15
15
|
end
|
16
16
|
|
17
|
-
def handle_exception(ex,
|
17
|
+
def handle_exception(ex, ctx = {})
|
18
18
|
Sidekiq.error_handlers.each do |handler|
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
Sidekiq.logger.error ex.backtrace.join("\n") unless ex.backtrace.nil?
|
25
|
-
end
|
19
|
+
handler.call(ex, ctx)
|
20
|
+
rescue => ex
|
21
|
+
Sidekiq.logger.error "!!! ERROR HANDLER THREW AN ERROR !!!"
|
22
|
+
Sidekiq.logger.error ex
|
23
|
+
Sidekiq.logger.error ex.backtrace.join("\n") unless ex.backtrace.nil?
|
26
24
|
end
|
27
25
|
end
|
28
26
|
end
|
@@ -1,12 +1,14 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
|
-
|
2
|
+
|
3
|
+
require "sidekiq/extensions/generic_proxy"
|
3
4
|
|
4
5
|
module Sidekiq
|
5
6
|
module Extensions
|
6
7
|
##
|
7
|
-
# Adds
|
8
|
-
# delivery to Sidekiq.
|
8
|
+
# Adds +delay+, +delay_for+ and +delay_until+ methods to ActionMailer to offload arbitrary email
|
9
|
+
# delivery to Sidekiq.
|
9
10
|
#
|
11
|
+
# @example
|
10
12
|
# UserMailer.delay.send_welcome_email(new_user)
|
11
13
|
# UserMailer.delay_for(5.days).send_welcome_email(new_user)
|
12
14
|
# UserMailer.delay_until(5.days.from_now).send_welcome_email(new_user)
|
@@ -19,39 +21,28 @@ module Sidekiq
|
|
19
21
|
# The email method can return nil, which causes ActionMailer to return
|
20
22
|
# an undeliverable empty message.
|
21
23
|
if msg
|
22
|
-
deliver(msg)
|
23
|
-
else
|
24
|
-
raise "#{target.name}##{method_name} returned an undeliverable mail object"
|
25
|
-
end
|
26
|
-
end
|
27
|
-
|
28
|
-
private
|
29
|
-
|
30
|
-
def deliver(msg)
|
31
|
-
if msg.respond_to?(:deliver_now)
|
32
|
-
# Rails 4.2/5.0
|
33
24
|
msg.deliver_now
|
34
25
|
else
|
35
|
-
#
|
36
|
-
msg.deliver
|
26
|
+
raise "#{target.name}##{method_name} returned an undeliverable mail object"
|
37
27
|
end
|
38
28
|
end
|
39
29
|
end
|
40
30
|
|
41
31
|
module ActionMailer
|
42
|
-
def sidekiq_delay(options={})
|
32
|
+
def sidekiq_delay(options = {})
|
43
33
|
Proxy.new(DelayedMailer, self, options)
|
44
34
|
end
|
45
|
-
|
46
|
-
|
35
|
+
|
36
|
+
def sidekiq_delay_for(interval, options = {})
|
37
|
+
Proxy.new(DelayedMailer, self, options.merge("at" => Time.now.to_f + interval.to_f))
|
47
38
|
end
|
48
|
-
|
49
|
-
|
39
|
+
|
40
|
+
def sidekiq_delay_until(timestamp, options = {})
|
41
|
+
Proxy.new(DelayedMailer, self, options.merge("at" => timestamp.to_f))
|
50
42
|
end
|
51
43
|
alias_method :delay, :sidekiq_delay
|
52
44
|
alias_method :delay_for, :sidekiq_delay_for
|
53
45
|
alias_method :delay_until, :sidekiq_delay_until
|
54
46
|
end
|
55
|
-
|
56
47
|
end
|
57
48
|
end
|
@@ -1,13 +1,15 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
|
-
|
2
|
+
|
3
|
+
require "sidekiq/extensions/generic_proxy"
|
3
4
|
|
4
5
|
module Sidekiq
|
5
6
|
module Extensions
|
6
7
|
##
|
7
|
-
# Adds
|
8
|
-
# execution to Sidekiq.
|
8
|
+
# Adds +delay+, +delay_for+ and +delay_until+ methods to ActiveRecord to offload instance method
|
9
|
+
# execution to Sidekiq.
|
9
10
|
#
|
10
|
-
#
|
11
|
+
# @example
|
12
|
+
# User.recent_signups.each { |user| user.delay.mark_as_awesome }
|
11
13
|
#
|
12
14
|
# Please note, this is not recommended as this will serialize the entire
|
13
15
|
# object to Redis. Your Sidekiq jobs should pass IDs, not entire instances.
|
@@ -22,19 +24,20 @@ module Sidekiq
|
|
22
24
|
end
|
23
25
|
|
24
26
|
module ActiveRecord
|
25
|
-
def sidekiq_delay(options={})
|
27
|
+
def sidekiq_delay(options = {})
|
26
28
|
Proxy.new(DelayedModel, self, options)
|
27
29
|
end
|
28
|
-
|
29
|
-
|
30
|
+
|
31
|
+
def sidekiq_delay_for(interval, options = {})
|
32
|
+
Proxy.new(DelayedModel, self, options.merge("at" => Time.now.to_f + interval.to_f))
|
30
33
|
end
|
31
|
-
|
32
|
-
|
34
|
+
|
35
|
+
def sidekiq_delay_until(timestamp, options = {})
|
36
|
+
Proxy.new(DelayedModel, self, options.merge("at" => timestamp.to_f))
|
33
37
|
end
|
34
38
|
alias_method :delay, :sidekiq_delay
|
35
39
|
alias_method :delay_for, :sidekiq_delay_for
|
36
40
|
alias_method :delay_until, :sidekiq_delay_until
|
37
41
|
end
|
38
|
-
|
39
42
|
end
|
40
43
|
end
|
@@ -1,14 +1,16 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
|
-
|
2
|
+
|
3
|
+
require "sidekiq/extensions/generic_proxy"
|
3
4
|
|
4
5
|
module Sidekiq
|
5
6
|
module Extensions
|
6
7
|
##
|
7
|
-
# Adds
|
8
|
-
# execution to Sidekiq.
|
8
|
+
# Adds `delay`, `delay_for` and `delay_until` methods to all Classes to offload class method
|
9
|
+
# execution to Sidekiq.
|
9
10
|
#
|
10
|
-
#
|
11
|
-
#
|
11
|
+
# @example
|
12
|
+
# User.delay.delete_inactive
|
13
|
+
# Wikipedia.delay.download_changes_for(Date.today)
|
12
14
|
#
|
13
15
|
class DelayedClass
|
14
16
|
include Sidekiq::Worker
|
@@ -20,20 +22,21 @@ module Sidekiq
|
|
20
22
|
end
|
21
23
|
|
22
24
|
module Klass
|
23
|
-
def sidekiq_delay(options={})
|
25
|
+
def sidekiq_delay(options = {})
|
24
26
|
Proxy.new(DelayedClass, self, options)
|
25
27
|
end
|
26
|
-
|
27
|
-
|
28
|
+
|
29
|
+
def sidekiq_delay_for(interval, options = {})
|
30
|
+
Proxy.new(DelayedClass, self, options.merge("at" => Time.now.to_f + interval.to_f))
|
28
31
|
end
|
29
|
-
|
30
|
-
|
32
|
+
|
33
|
+
def sidekiq_delay_until(timestamp, options = {})
|
34
|
+
Proxy.new(DelayedClass, self, options.merge("at" => timestamp.to_f))
|
31
35
|
end
|
32
36
|
alias_method :delay, :sidekiq_delay
|
33
37
|
alias_method :delay_for, :sidekiq_delay_for
|
34
38
|
alias_method :delay_until, :sidekiq_delay_until
|
35
39
|
end
|
36
|
-
|
37
40
|
end
|
38
41
|
end
|
39
42
|
|
@@ -1,12 +1,13 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
|
-
|
2
|
+
|
3
|
+
require "yaml"
|
3
4
|
|
4
5
|
module Sidekiq
|
5
6
|
module Extensions
|
6
7
|
SIZE_LIMIT = 8_192
|
7
8
|
|
8
9
|
class Proxy < BasicObject
|
9
|
-
def initialize(performable, target, options={})
|
10
|
+
def initialize(performable, target, options = {})
|
10
11
|
@performable = performable
|
11
12
|
@target = target
|
12
13
|
@opts = options
|
@@ -23,9 +24,10 @@ module Sidekiq
|
|
23
24
|
if marshalled.size > SIZE_LIMIT
|
24
25
|
::Sidekiq.logger.warn { "#{@target}.#{name} job argument is #{marshalled.bytesize} bytes, you should refactor it to reduce the size" }
|
25
26
|
end
|
26
|
-
@performable.client_push({
|
27
|
+
@performable.client_push({"class" => @performable,
|
28
|
+
"args" => [marshalled],
|
29
|
+
"display_class" => "#{@target}.#{name}"}.merge(@opts))
|
27
30
|
end
|
28
31
|
end
|
29
|
-
|
30
32
|
end
|
31
33
|
end
|
data/lib/sidekiq/fetch.rb
CHANGED
@@ -1,5 +1,6 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
|
-
|
2
|
+
|
3
|
+
require "sidekiq"
|
3
4
|
|
4
5
|
module Sidekiq
|
5
6
|
class BasicFetch
|
@@ -7,68 +8,60 @@ module Sidekiq
|
|
7
8
|
# can check if the process is shutting down.
|
8
9
|
TIMEOUT = 2
|
9
10
|
|
10
|
-
UnitOfWork = Struct.new(:queue, :job)
|
11
|
+
UnitOfWork = Struct.new(:queue, :job) {
|
11
12
|
def acknowledge
|
12
13
|
# nothing to do
|
13
14
|
end
|
14
15
|
|
15
16
|
def queue_name
|
16
|
-
queue.
|
17
|
+
queue.delete_prefix("queue:")
|
17
18
|
end
|
18
19
|
|
19
20
|
def requeue
|
20
21
|
Sidekiq.redis do |conn|
|
21
|
-
conn.rpush(
|
22
|
+
conn.rpush(queue, job)
|
22
23
|
end
|
23
24
|
end
|
24
|
-
|
25
|
+
}
|
25
26
|
|
26
27
|
def initialize(options)
|
27
|
-
|
28
|
-
@
|
28
|
+
raise ArgumentError, "missing queue list" unless options[:queues]
|
29
|
+
@options = options
|
30
|
+
@strictly_ordered_queues = !!@options[:strict]
|
31
|
+
@queues = @options[:queues].map { |q| "queue:#{q}" }
|
29
32
|
if @strictly_ordered_queues
|
30
|
-
@queues
|
33
|
+
@queues.uniq!
|
31
34
|
@queues << TIMEOUT
|
32
35
|
end
|
33
36
|
end
|
34
37
|
|
35
38
|
def retrieve_work
|
36
|
-
|
37
|
-
|
38
|
-
|
39
|
-
|
40
|
-
|
41
|
-
|
42
|
-
# data from the first queue that has pending elements. We
|
43
|
-
# recreate the queue command each time we invoke Redis#brpop
|
44
|
-
# to honor weights and avoid queue starvation.
|
45
|
-
def queues_cmd
|
46
|
-
if @strictly_ordered_queues
|
47
|
-
@queues
|
48
|
-
else
|
49
|
-
queues = @queues.shuffle.uniq
|
50
|
-
queues << TIMEOUT
|
51
|
-
queues
|
39
|
+
qs = queues_cmd
|
40
|
+
# 4825 Sidekiq Pro with all queues paused will return an
|
41
|
+
# empty set of queues with a trailing TIMEOUT value.
|
42
|
+
if qs.size <= 1
|
43
|
+
sleep(TIMEOUT)
|
44
|
+
return nil
|
52
45
|
end
|
53
|
-
end
|
54
46
|
|
47
|
+
work = Sidekiq.redis { |conn| conn.brpop(*qs) }
|
48
|
+
UnitOfWork.new(*work) if work
|
49
|
+
end
|
55
50
|
|
56
|
-
|
57
|
-
# an instance method will make it async to the Fetcher actor
|
58
|
-
def self.bulk_requeue(inprogress, options)
|
51
|
+
def bulk_requeue(inprogress, options)
|
59
52
|
return if inprogress.empty?
|
60
53
|
|
61
54
|
Sidekiq.logger.debug { "Re-queueing terminated jobs" }
|
62
55
|
jobs_to_requeue = {}
|
63
56
|
inprogress.each do |unit_of_work|
|
64
|
-
jobs_to_requeue[unit_of_work.
|
65
|
-
jobs_to_requeue[unit_of_work.
|
57
|
+
jobs_to_requeue[unit_of_work.queue] ||= []
|
58
|
+
jobs_to_requeue[unit_of_work.queue] << unit_of_work.job
|
66
59
|
end
|
67
60
|
|
68
61
|
Sidekiq.redis do |conn|
|
69
|
-
conn.pipelined do
|
62
|
+
conn.pipelined do |pipeline|
|
70
63
|
jobs_to_requeue.each do |queue, jobs|
|
71
|
-
|
64
|
+
pipeline.rpush(queue, jobs)
|
72
65
|
end
|
73
66
|
end
|
74
67
|
end
|
@@ -77,5 +70,20 @@ module Sidekiq
|
|
77
70
|
Sidekiq.logger.warn("Failed to requeue #{inprogress.size} jobs: #{ex.message}")
|
78
71
|
end
|
79
72
|
|
73
|
+
# Creating the Redis#brpop command takes into account any
|
74
|
+
# configured queue weights. By default Redis#brpop returns
|
75
|
+
# data from the first queue that has pending elements. We
|
76
|
+
# recreate the queue command each time we invoke Redis#brpop
|
77
|
+
# to honor weights and avoid queue starvation.
|
78
|
+
def queues_cmd
|
79
|
+
if @strictly_ordered_queues
|
80
|
+
@queues
|
81
|
+
else
|
82
|
+
permute = @queues.shuffle
|
83
|
+
permute.uniq!
|
84
|
+
permute << TIMEOUT
|
85
|
+
permute
|
86
|
+
end
|
87
|
+
end
|
80
88
|
end
|
81
89
|
end
|
data/lib/sidekiq/job.rb
ADDED
@@ -0,0 +1,13 @@
|
|
1
|
+
require "sidekiq/worker"
|
2
|
+
|
3
|
+
module Sidekiq
|
4
|
+
# Sidekiq::Job is a new alias for Sidekiq::Worker as of Sidekiq 6.3.0.
|
5
|
+
# Use `include Sidekiq::Job` rather than `include Sidekiq::Worker`.
|
6
|
+
#
|
7
|
+
# The term "worker" is too generic and overly confusing, used in several
|
8
|
+
# different contexts meaning different things. Many people call a Sidekiq
|
9
|
+
# process a "worker". Some people call the thread that executes jobs a
|
10
|
+
# "worker". This change brings Sidekiq closer to ActiveJob where your job
|
11
|
+
# classes extend ApplicationJob.
|
12
|
+
Job = Worker
|
13
|
+
end
|
data/lib/sidekiq/job_logger.rb
CHANGED
@@ -1,25 +1,51 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
|
+
|
2
3
|
module Sidekiq
|
3
4
|
class JobLogger
|
5
|
+
def initialize(logger = Sidekiq.logger)
|
6
|
+
@logger = logger
|
7
|
+
end
|
4
8
|
|
5
9
|
def call(item, queue)
|
6
10
|
start = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
|
7
|
-
logger.info("start")
|
11
|
+
@logger.info("start")
|
12
|
+
|
8
13
|
yield
|
9
|
-
|
14
|
+
|
15
|
+
Sidekiq::Context.add(:elapsed, elapsed(start))
|
16
|
+
@logger.info("done")
|
10
17
|
rescue Exception
|
11
|
-
|
18
|
+
Sidekiq::Context.add(:elapsed, elapsed(start))
|
19
|
+
@logger.info("fail")
|
20
|
+
|
12
21
|
raise
|
13
22
|
end
|
14
23
|
|
24
|
+
def prepare(job_hash, &block)
|
25
|
+
# If we're using a wrapper class, like ActiveJob, use the "wrapped"
|
26
|
+
# attribute to expose the underlying thing.
|
27
|
+
h = {
|
28
|
+
class: job_hash["display_class"] || job_hash["wrapped"] || job_hash["class"],
|
29
|
+
jid: job_hash["jid"]
|
30
|
+
}
|
31
|
+
h[:bid] = job_hash["bid"] if job_hash.has_key?("bid")
|
32
|
+
h[:tags] = job_hash["tags"] if job_hash.has_key?("tags")
|
33
|
+
|
34
|
+
Thread.current[:sidekiq_context] = h
|
35
|
+
level = job_hash["log_level"]
|
36
|
+
if level
|
37
|
+
@logger.log_at(level, &block)
|
38
|
+
else
|
39
|
+
yield
|
40
|
+
end
|
41
|
+
ensure
|
42
|
+
Thread.current[:sidekiq_context] = nil
|
43
|
+
end
|
44
|
+
|
15
45
|
private
|
16
46
|
|
17
47
|
def elapsed(start)
|
18
48
|
(::Process.clock_gettime(::Process::CLOCK_MONOTONIC) - start).round(3)
|
19
49
|
end
|
20
|
-
|
21
|
-
def logger
|
22
|
-
Sidekiq.logger
|
23
|
-
end
|
24
50
|
end
|
25
51
|
end
|