sidekiq 4.2.2 → 6.3.1
Sign up to get free protection for your applications and to get access to all the features.
Potentially problematic release.
This version of sidekiq might be problematic. Click here for more details.
- checksums.yaml +5 -5
- data/Changes.md +516 -0
- data/LICENSE +2 -2
- data/README.md +23 -36
- data/bin/sidekiq +26 -2
- data/bin/sidekiqload +28 -38
- data/bin/sidekiqmon +8 -0
- data/lib/generators/sidekiq/templates/worker_spec.rb.erb +1 -1
- data/lib/generators/sidekiq/templates/worker_test.rb.erb +2 -2
- data/lib/generators/sidekiq/worker_generator.rb +21 -13
- data/lib/sidekiq/api.rb +401 -243
- data/lib/sidekiq/cli.rb +228 -212
- data/lib/sidekiq/client.rb +76 -53
- data/lib/sidekiq/delay.rb +41 -0
- data/lib/sidekiq/exception_handler.rb +12 -16
- data/lib/sidekiq/extensions/action_mailer.rb +13 -22
- data/lib/sidekiq/extensions/active_record.rb +13 -10
- data/lib/sidekiq/extensions/class_methods.rb +14 -11
- data/lib/sidekiq/extensions/generic_proxy.rb +12 -4
- data/lib/sidekiq/fetch.rb +39 -31
- data/lib/sidekiq/job.rb +13 -0
- data/lib/sidekiq/job_logger.rb +63 -0
- data/lib/sidekiq/job_retry.rb +259 -0
- data/lib/sidekiq/launcher.rb +170 -71
- data/lib/sidekiq/logger.rb +166 -0
- data/lib/sidekiq/manager.rb +17 -20
- data/lib/sidekiq/middleware/chain.rb +20 -8
- data/lib/sidekiq/middleware/current_attributes.rb +52 -0
- data/lib/sidekiq/middleware/i18n.rb +5 -7
- data/lib/sidekiq/monitor.rb +133 -0
- data/lib/sidekiq/paginator.rb +18 -14
- data/lib/sidekiq/processor.rb +169 -78
- data/lib/sidekiq/rails.rb +41 -36
- data/lib/sidekiq/redis_connection.rb +65 -20
- data/lib/sidekiq/scheduled.rb +85 -34
- data/lib/sidekiq/sd_notify.rb +149 -0
- data/lib/sidekiq/systemd.rb +24 -0
- data/lib/sidekiq/testing/inline.rb +2 -1
- data/lib/sidekiq/testing.rb +52 -26
- data/lib/sidekiq/util.rb +48 -15
- data/lib/sidekiq/version.rb +2 -1
- data/lib/sidekiq/web/action.rb +15 -17
- data/lib/sidekiq/web/application.rb +114 -92
- data/lib/sidekiq/web/csrf_protection.rb +180 -0
- data/lib/sidekiq/web/helpers.rb +151 -83
- data/lib/sidekiq/web/router.rb +27 -19
- data/lib/sidekiq/web.rb +85 -76
- data/lib/sidekiq/worker.rb +233 -43
- data/lib/sidekiq.rb +88 -64
- data/sidekiq.gemspec +24 -22
- data/web/assets/images/apple-touch-icon.png +0 -0
- data/web/assets/javascripts/application.js +86 -59
- data/web/assets/javascripts/dashboard.js +81 -85
- data/web/assets/stylesheets/application-dark.css +147 -0
- data/web/assets/stylesheets/application-rtl.css +242 -0
- data/web/assets/stylesheets/application.css +319 -141
- data/web/assets/stylesheets/bootstrap-rtl.min.css +9 -0
- data/web/assets/stylesheets/bootstrap.css +2 -2
- data/web/locales/ar.yml +87 -0
- data/web/locales/de.yml +14 -2
- data/web/locales/en.yml +8 -1
- data/web/locales/es.yml +22 -5
- data/web/locales/fa.yml +80 -0
- data/web/locales/fr.yml +10 -3
- data/web/locales/he.yml +79 -0
- data/web/locales/ja.yml +12 -4
- data/web/locales/lt.yml +83 -0
- data/web/locales/pl.yml +4 -4
- data/web/locales/ru.yml +4 -0
- data/web/locales/ur.yml +80 -0
- data/web/locales/vi.yml +83 -0
- data/web/views/_footer.erb +5 -2
- data/web/views/_job_info.erb +4 -3
- data/web/views/_nav.erb +4 -18
- data/web/views/_paging.erb +1 -1
- data/web/views/_poll_link.erb +2 -5
- data/web/views/_summary.erb +7 -7
- data/web/views/busy.erb +60 -22
- data/web/views/dashboard.erb +23 -15
- data/web/views/dead.erb +3 -3
- data/web/views/layout.erb +14 -3
- data/web/views/morgue.erb +19 -12
- data/web/views/queue.erb +24 -14
- data/web/views/queues.erb +14 -4
- data/web/views/retries.erb +22 -13
- data/web/views/retry.erb +4 -4
- data/web/views/scheduled.erb +7 -4
- metadata +44 -194
- data/.github/contributing.md +0 -32
- data/.github/issue_template.md +0 -4
- data/.gitignore +0 -12
- data/.travis.yml +0 -12
- data/3.0-Upgrade.md +0 -70
- data/4.0-Upgrade.md +0 -53
- data/COMM-LICENSE +0 -95
- data/Ent-Changes.md +0 -146
- data/Gemfile +0 -29
- data/Pro-2.0-Upgrade.md +0 -138
- data/Pro-3.0-Upgrade.md +0 -44
- data/Pro-Changes.md +0 -570
- data/Rakefile +0 -9
- data/bin/sidekiqctl +0 -99
- data/code_of_conduct.md +0 -50
- data/lib/sidekiq/core_ext.rb +0 -106
- data/lib/sidekiq/logging.rb +0 -106
- data/lib/sidekiq/middleware/server/active_record.rb +0 -13
- data/lib/sidekiq/middleware/server/logging.rb +0 -40
- data/lib/sidekiq/middleware/server/retry_jobs.rb +0 -205
- data/test/config.yml +0 -9
- data/test/env_based_config.yml +0 -11
- data/test/fake_env.rb +0 -1
- data/test/fixtures/en.yml +0 -2
- data/test/helper.rb +0 -75
- data/test/test_actors.rb +0 -138
- data/test/test_api.rb +0 -528
- data/test/test_cli.rb +0 -418
- data/test/test_client.rb +0 -266
- data/test/test_exception_handler.rb +0 -56
- data/test/test_extensions.rb +0 -127
- data/test/test_fetch.rb +0 -50
- data/test/test_launcher.rb +0 -95
- data/test/test_logging.rb +0 -35
- data/test/test_manager.rb +0 -50
- data/test/test_middleware.rb +0 -158
- data/test/test_processor.rb +0 -201
- data/test/test_rails.rb +0 -22
- data/test/test_redis_connection.rb +0 -132
- data/test/test_retry.rb +0 -326
- data/test/test_retry_exhausted.rb +0 -149
- data/test/test_scheduled.rb +0 -115
- data/test/test_scheduling.rb +0 -50
- data/test/test_sidekiq.rb +0 -107
- data/test/test_testing.rb +0 -143
- data/test/test_testing_fake.rb +0 -357
- data/test/test_testing_inline.rb +0 -94
- data/test/test_util.rb +0 -13
- data/test/test_web.rb +0 -666
- data/test/test_web_helpers.rb +0 -54
data/lib/sidekiq/client.rb
CHANGED
@@ -1,10 +1,10 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
|
-
|
3
|
-
require
|
2
|
+
|
3
|
+
require "securerandom"
|
4
|
+
require "sidekiq/middleware/chain"
|
4
5
|
|
5
6
|
module Sidekiq
|
6
7
|
class Client
|
7
|
-
|
8
8
|
##
|
9
9
|
# Define client-side middleware:
|
10
10
|
#
|
@@ -19,7 +19,7 @@ module Sidekiq
|
|
19
19
|
#
|
20
20
|
def middleware(&block)
|
21
21
|
@chain ||= Sidekiq.client_middleware
|
22
|
-
if
|
22
|
+
if block
|
23
23
|
@chain = @chain.dup
|
24
24
|
yield @chain
|
25
25
|
end
|
@@ -38,7 +38,7 @@ module Sidekiq
|
|
38
38
|
# Generally this is only needed for very large Sidekiq installs processing
|
39
39
|
# thousands of jobs per second. I don't recommend sharding unless you
|
40
40
|
# cannot scale any other way (e.g. splitting your app into smaller apps).
|
41
|
-
def initialize(redis_pool=nil)
|
41
|
+
def initialize(redis_pool = nil)
|
42
42
|
@redis_pool = redis_pool || Thread.current[:sidekiq_via_pool] || Sidekiq.redis_pool
|
43
43
|
end
|
44
44
|
|
@@ -48,9 +48,15 @@ module Sidekiq
|
|
48
48
|
# queue - the named queue to use, default 'default'
|
49
49
|
# class - the worker class to call, required
|
50
50
|
# args - an array of simple arguments to the perform method, must be JSON-serializable
|
51
|
+
# at - timestamp to schedule the job (optional), must be Numeric (e.g. Time.now.to_f)
|
51
52
|
# retry - whether to retry this job if it fails, default true or an integer number of retries
|
52
53
|
# backtrace - whether to save any error backtrace, default false
|
53
54
|
#
|
55
|
+
# If class is set to the class name, the jobs' options will be based on Sidekiq's default
|
56
|
+
# worker options. Otherwise, they will be based on the job class's options.
|
57
|
+
#
|
58
|
+
# Any options valid for a worker class's sidekiq_options are also available here.
|
59
|
+
#
|
54
60
|
# All options must be strings, not symbols. NB: because we are serializing to JSON, all
|
55
61
|
# symbols in 'args' will be converted to strings. Note that +backtrace: true+ can take quite a bit of
|
56
62
|
# space in Redis; a large volume of failing jobs can start Redis swapping if you aren't careful.
|
@@ -62,18 +68,19 @@ module Sidekiq
|
|
62
68
|
#
|
63
69
|
def push(item)
|
64
70
|
normed = normalize_item(item)
|
65
|
-
payload = process_single(item[
|
71
|
+
payload = process_single(item["class"], normed)
|
66
72
|
|
67
73
|
if payload
|
68
74
|
raw_push([payload])
|
69
|
-
payload[
|
75
|
+
payload["jid"]
|
70
76
|
end
|
71
77
|
end
|
72
78
|
|
73
79
|
##
|
74
|
-
# Push a large number of jobs to Redis.
|
75
|
-
#
|
76
|
-
#
|
80
|
+
# Push a large number of jobs to Redis. This method cuts out the redis
|
81
|
+
# network round trip latency. I wouldn't recommend pushing more than
|
82
|
+
# 1000 per call but YMMV based on network quality, size of job args, etc.
|
83
|
+
# A large number of jobs can cause a bit of Redis command processing latency.
|
77
84
|
#
|
78
85
|
# Takes the same arguments as #push except that args is expected to be
|
79
86
|
# an Array of Arrays. All other keys are duplicated for each job. Each job
|
@@ -83,19 +90,25 @@ module Sidekiq
|
|
83
90
|
# Returns an array of the of pushed jobs' jids. The number of jobs pushed can be less
|
84
91
|
# than the number given if the middleware stopped processing for one or more jobs.
|
85
92
|
def push_bulk(items)
|
86
|
-
|
87
|
-
|
88
|
-
|
93
|
+
args = items["args"]
|
94
|
+
raise ArgumentError, "Bulk arguments must be an Array of Arrays: [[1], [2]]" unless args.is_a?(Array) && args.all?(Array)
|
95
|
+
return [] if args.empty? # no jobs to push
|
96
|
+
|
97
|
+
at = items.delete("at")
|
98
|
+
raise ArgumentError, "Job 'at' must be a Numeric or an Array of Numeric timestamps" if at && (Array(at).empty? || !Array(at).all? { |entry| entry.is_a?(Numeric) })
|
99
|
+
raise ArgumentError, "Job 'at' Array must have same size as 'args' Array" if at.is_a?(Array) && at.size != args.size
|
89
100
|
|
90
101
|
normed = normalize_item(items)
|
91
|
-
payloads =
|
92
|
-
copy = normed.merge(
|
93
|
-
|
94
|
-
|
95
|
-
|
96
|
-
|
97
|
-
|
98
|
-
|
102
|
+
payloads = args.map.with_index { |job_args, index|
|
103
|
+
copy = normed.merge("args" => job_args, "jid" => SecureRandom.hex(12), "enqueued_at" => Time.now.to_f)
|
104
|
+
copy["at"] = (at.is_a?(Array) ? at[index] : at) if at
|
105
|
+
|
106
|
+
result = process_single(items["class"], copy)
|
107
|
+
result || nil
|
108
|
+
}.compact
|
109
|
+
|
110
|
+
raw_push(payloads) unless payloads.empty?
|
111
|
+
payloads.collect { |payload| payload["jid"] }
|
99
112
|
end
|
100
113
|
|
101
114
|
# Allows sharding of jobs across any number of Redis instances. All jobs
|
@@ -113,15 +126,13 @@ module Sidekiq
|
|
113
126
|
def self.via(pool)
|
114
127
|
raise ArgumentError, "No pool given" if pool.nil?
|
115
128
|
current_sidekiq_pool = Thread.current[:sidekiq_via_pool]
|
116
|
-
raise RuntimeError, "Sidekiq::Client.via is not re-entrant" if current_sidekiq_pool && current_sidekiq_pool != pool
|
117
129
|
Thread.current[:sidekiq_via_pool] = pool
|
118
130
|
yield
|
119
131
|
ensure
|
120
|
-
Thread.current[:sidekiq_via_pool] =
|
132
|
+
Thread.current[:sidekiq_via_pool] = current_sidekiq_pool
|
121
133
|
end
|
122
134
|
|
123
135
|
class << self
|
124
|
-
|
125
136
|
def push(item)
|
126
137
|
new.push(item)
|
127
138
|
end
|
@@ -139,14 +150,14 @@ module Sidekiq
|
|
139
150
|
# Messages are enqueued to the 'default' queue.
|
140
151
|
#
|
141
152
|
def enqueue(klass, *args)
|
142
|
-
klass.client_push(
|
153
|
+
klass.client_push("class" => klass, "args" => args)
|
143
154
|
end
|
144
155
|
|
145
156
|
# Example usage:
|
146
157
|
# Sidekiq::Client.enqueue_to(:queue_name, MyWorker, 'foo', 1, :bat => 'bar')
|
147
158
|
#
|
148
159
|
def enqueue_to(queue, klass, *args)
|
149
|
-
klass.client_push(
|
160
|
+
klass.client_push("queue" => queue, "class" => klass, "args" => args)
|
150
161
|
end
|
151
162
|
|
152
163
|
# Example usage:
|
@@ -157,8 +168,8 @@ module Sidekiq
|
|
157
168
|
now = Time.now.to_f
|
158
169
|
ts = (int < 1_000_000_000 ? now + int : int)
|
159
170
|
|
160
|
-
item = {
|
161
|
-
item.delete(
|
171
|
+
item = {"class" => klass, "args" => args, "at" => ts, "queue" => queue}
|
172
|
+
item.delete("at") if ts <= now
|
162
173
|
|
163
174
|
klass.client_push(item)
|
164
175
|
end
|
@@ -175,7 +186,7 @@ module Sidekiq
|
|
175
186
|
|
176
187
|
def raw_push(payloads)
|
177
188
|
@redis_pool.with do |conn|
|
178
|
-
conn.
|
189
|
+
conn.pipelined do
|
179
190
|
atomic_push(conn, payloads)
|
180
191
|
end
|
181
192
|
end
|
@@ -183,50 +194,62 @@ module Sidekiq
|
|
183
194
|
end
|
184
195
|
|
185
196
|
def atomic_push(conn, payloads)
|
186
|
-
if payloads.first
|
187
|
-
conn.zadd(
|
188
|
-
at = hash.delete(
|
197
|
+
if payloads.first.key?("at")
|
198
|
+
conn.zadd("schedule", payloads.map { |hash|
|
199
|
+
at = hash.delete("at").to_s
|
189
200
|
[at, Sidekiq.dump_json(hash)]
|
190
|
-
|
201
|
+
})
|
191
202
|
else
|
192
|
-
|
203
|
+
queue = payloads.first["queue"]
|
193
204
|
now = Time.now.to_f
|
194
|
-
to_push = payloads.map
|
195
|
-
entry[
|
205
|
+
to_push = payloads.map { |entry|
|
206
|
+
entry["enqueued_at"] = now
|
196
207
|
Sidekiq.dump_json(entry)
|
197
|
-
|
198
|
-
conn.sadd(
|
199
|
-
conn.lpush("queue:#{
|
208
|
+
}
|
209
|
+
conn.sadd("queues", queue)
|
210
|
+
conn.lpush("queue:#{queue}", to_push)
|
200
211
|
end
|
201
212
|
end
|
202
213
|
|
203
214
|
def process_single(worker_class, item)
|
204
|
-
queue = item[
|
215
|
+
queue = item["queue"]
|
205
216
|
|
206
217
|
middleware.invoke(worker_class, item, queue, @redis_pool) do
|
207
218
|
item
|
208
219
|
end
|
209
220
|
end
|
210
221
|
|
222
|
+
def validate(item)
|
223
|
+
raise(ArgumentError, "Job must be a Hash with 'class' and 'args' keys: `#{item}`") unless item.is_a?(Hash) && item.key?("class") && item.key?("args")
|
224
|
+
raise(ArgumentError, "Job args must be an Array: `#{item}`") unless item["args"].is_a?(Array)
|
225
|
+
raise(ArgumentError, "Job class must be either a Class or String representation of the class name: `#{item}`") unless item["class"].is_a?(Class) || item["class"].is_a?(String)
|
226
|
+
raise(ArgumentError, "Job 'at' must be a Numeric timestamp: `#{item}`") if item.key?("at") && !item["at"].is_a?(Numeric)
|
227
|
+
raise(ArgumentError, "Job tags must be an Array: `#{item}`") if item["tags"] && !item["tags"].is_a?(Array)
|
228
|
+
end
|
229
|
+
|
211
230
|
def normalize_item(item)
|
212
|
-
|
213
|
-
raise(ArgumentError, "
|
214
|
-
|
215
|
-
#
|
216
|
-
|
217
|
-
normalized_hash(item[
|
218
|
-
|
219
|
-
|
220
|
-
|
221
|
-
item[
|
222
|
-
|
223
|
-
item[
|
231
|
+
validate(item)
|
232
|
+
# raise(ArgumentError, "Arguments must be native JSON types, see https://github.com/mperham/sidekiq/wiki/Best-Practices") unless JSON.load(JSON.dump(item['args'])) == item['args']
|
233
|
+
|
234
|
+
# merge in the default sidekiq_options for the item's class and/or wrapped element
|
235
|
+
# this allows ActiveJobs to control sidekiq_options too.
|
236
|
+
defaults = normalized_hash(item["class"])
|
237
|
+
defaults = defaults.merge(item["wrapped"].get_sidekiq_options) if item["wrapped"].respond_to?("get_sidekiq_options")
|
238
|
+
item = defaults.merge(item)
|
239
|
+
|
240
|
+
raise(ArgumentError, "Job must include a valid queue name") if item["queue"].nil? || item["queue"] == ""
|
241
|
+
|
242
|
+
item["class"] = item["class"].to_s
|
243
|
+
item["queue"] = item["queue"].to_s
|
244
|
+
item["jid"] ||= SecureRandom.hex(12)
|
245
|
+
item["created_at"] ||= Time.now.to_f
|
246
|
+
|
224
247
|
item
|
225
248
|
end
|
226
249
|
|
227
250
|
def normalized_hash(item_class)
|
228
251
|
if item_class.is_a?(Class)
|
229
|
-
raise(ArgumentError, "Message must include a Sidekiq::Worker class, not class name: #{item_class.ancestors.inspect}")
|
252
|
+
raise(ArgumentError, "Message must include a Sidekiq::Worker class, not class name: #{item_class.ancestors.inspect}") unless item_class.respond_to?("get_sidekiq_options")
|
230
253
|
item_class.get_sidekiq_options
|
231
254
|
else
|
232
255
|
Sidekiq.default_worker_options
|
@@ -0,0 +1,41 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Sidekiq
|
4
|
+
module Extensions
|
5
|
+
def self.enable_delay!
|
6
|
+
if defined?(::ActiveSupport)
|
7
|
+
require "sidekiq/extensions/active_record"
|
8
|
+
require "sidekiq/extensions/action_mailer"
|
9
|
+
|
10
|
+
# Need to patch Psych so it can autoload classes whose names are serialized
|
11
|
+
# in the delayed YAML.
|
12
|
+
Psych::Visitors::ToRuby.prepend(Sidekiq::Extensions::PsychAutoload)
|
13
|
+
|
14
|
+
ActiveSupport.on_load(:active_record) do
|
15
|
+
include Sidekiq::Extensions::ActiveRecord
|
16
|
+
end
|
17
|
+
ActiveSupport.on_load(:action_mailer) do
|
18
|
+
extend Sidekiq::Extensions::ActionMailer
|
19
|
+
end
|
20
|
+
end
|
21
|
+
|
22
|
+
require "sidekiq/extensions/class_methods"
|
23
|
+
Module.__send__(:include, Sidekiq::Extensions::Klass)
|
24
|
+
end
|
25
|
+
|
26
|
+
module PsychAutoload
|
27
|
+
def resolve_class(klass_name)
|
28
|
+
return nil if !klass_name || klass_name.empty?
|
29
|
+
# constantize
|
30
|
+
names = klass_name.split("::")
|
31
|
+
names.shift if names.empty? || names.first.empty?
|
32
|
+
|
33
|
+
names.inject(Object) do |constant, name|
|
34
|
+
constant.const_defined?(name) ? constant.const_get(name) : constant.const_missing(name)
|
35
|
+
end
|
36
|
+
rescue NameError
|
37
|
+
super
|
38
|
+
end
|
39
|
+
end
|
40
|
+
end
|
41
|
+
end
|
@@ -1,31 +1,27 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
|
-
|
2
|
+
|
3
|
+
require "sidekiq"
|
3
4
|
|
4
5
|
module Sidekiq
|
5
6
|
module ExceptionHandler
|
6
|
-
|
7
7
|
class Logger
|
8
|
-
def call(ex,
|
9
|
-
Sidekiq.logger.warn(Sidekiq.dump_json(
|
10
|
-
Sidekiq.logger.warn
|
11
|
-
Sidekiq.logger.warn
|
8
|
+
def call(ex, ctx)
|
9
|
+
Sidekiq.logger.warn(Sidekiq.dump_json(ctx)) unless ctx.empty?
|
10
|
+
Sidekiq.logger.warn("#{ex.class.name}: #{ex.message}")
|
11
|
+
Sidekiq.logger.warn(ex.backtrace.join("\n")) unless ex.backtrace.nil?
|
12
12
|
end
|
13
13
|
|
14
|
-
# Set up default handler which just logs the error
|
15
14
|
Sidekiq.error_handlers << Sidekiq::ExceptionHandler::Logger.new
|
16
15
|
end
|
17
16
|
|
18
|
-
def handle_exception(ex,
|
17
|
+
def handle_exception(ex, ctx = {})
|
19
18
|
Sidekiq.error_handlers.each do |handler|
|
20
|
-
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
|
25
|
-
Sidekiq.logger.error ex.backtrace.join("\n") unless ex.backtrace.nil?
|
26
|
-
end
|
19
|
+
handler.call(ex, ctx)
|
20
|
+
rescue => ex
|
21
|
+
Sidekiq.logger.error "!!! ERROR HANDLER THREW AN ERROR !!!"
|
22
|
+
Sidekiq.logger.error ex
|
23
|
+
Sidekiq.logger.error ex.backtrace.join("\n") unless ex.backtrace.nil?
|
27
24
|
end
|
28
25
|
end
|
29
|
-
|
30
26
|
end
|
31
27
|
end
|
@@ -1,12 +1,14 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
|
-
|
2
|
+
|
3
|
+
require "sidekiq/extensions/generic_proxy"
|
3
4
|
|
4
5
|
module Sidekiq
|
5
6
|
module Extensions
|
6
7
|
##
|
7
|
-
# Adds
|
8
|
-
# delivery to Sidekiq.
|
8
|
+
# Adds +delay+, +delay_for+ and +delay_until+ methods to ActionMailer to offload arbitrary email
|
9
|
+
# delivery to Sidekiq.
|
9
10
|
#
|
11
|
+
# @example
|
10
12
|
# UserMailer.delay.send_welcome_email(new_user)
|
11
13
|
# UserMailer.delay_for(5.days).send_welcome_email(new_user)
|
12
14
|
# UserMailer.delay_until(5.days.from_now).send_welcome_email(new_user)
|
@@ -19,39 +21,28 @@ module Sidekiq
|
|
19
21
|
# The email method can return nil, which causes ActionMailer to return
|
20
22
|
# an undeliverable empty message.
|
21
23
|
if msg
|
22
|
-
deliver(msg)
|
23
|
-
else
|
24
|
-
raise "#{target.name}##{method_name} returned an undeliverable mail object"
|
25
|
-
end
|
26
|
-
end
|
27
|
-
|
28
|
-
private
|
29
|
-
|
30
|
-
def deliver(msg)
|
31
|
-
if msg.respond_to?(:deliver_now)
|
32
|
-
# Rails 4.2/5.0
|
33
24
|
msg.deliver_now
|
34
25
|
else
|
35
|
-
#
|
36
|
-
msg.deliver
|
26
|
+
raise "#{target.name}##{method_name} returned an undeliverable mail object"
|
37
27
|
end
|
38
28
|
end
|
39
29
|
end
|
40
30
|
|
41
31
|
module ActionMailer
|
42
|
-
def sidekiq_delay(options={})
|
32
|
+
def sidekiq_delay(options = {})
|
43
33
|
Proxy.new(DelayedMailer, self, options)
|
44
34
|
end
|
45
|
-
|
46
|
-
|
35
|
+
|
36
|
+
def sidekiq_delay_for(interval, options = {})
|
37
|
+
Proxy.new(DelayedMailer, self, options.merge("at" => Time.now.to_f + interval.to_f))
|
47
38
|
end
|
48
|
-
|
49
|
-
|
39
|
+
|
40
|
+
def sidekiq_delay_until(timestamp, options = {})
|
41
|
+
Proxy.new(DelayedMailer, self, options.merge("at" => timestamp.to_f))
|
50
42
|
end
|
51
43
|
alias_method :delay, :sidekiq_delay
|
52
44
|
alias_method :delay_for, :sidekiq_delay_for
|
53
45
|
alias_method :delay_until, :sidekiq_delay_until
|
54
46
|
end
|
55
|
-
|
56
47
|
end
|
57
48
|
end
|
@@ -1,13 +1,15 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
|
-
|
2
|
+
|
3
|
+
require "sidekiq/extensions/generic_proxy"
|
3
4
|
|
4
5
|
module Sidekiq
|
5
6
|
module Extensions
|
6
7
|
##
|
7
|
-
# Adds
|
8
|
-
# execution to Sidekiq.
|
8
|
+
# Adds +delay+, +delay_for+ and +delay_until+ methods to ActiveRecord to offload instance method
|
9
|
+
# execution to Sidekiq.
|
9
10
|
#
|
10
|
-
#
|
11
|
+
# @example
|
12
|
+
# User.recent_signups.each { |user| user.delay.mark_as_awesome }
|
11
13
|
#
|
12
14
|
# Please note, this is not recommended as this will serialize the entire
|
13
15
|
# object to Redis. Your Sidekiq jobs should pass IDs, not entire instances.
|
@@ -22,19 +24,20 @@ module Sidekiq
|
|
22
24
|
end
|
23
25
|
|
24
26
|
module ActiveRecord
|
25
|
-
def sidekiq_delay(options={})
|
27
|
+
def sidekiq_delay(options = {})
|
26
28
|
Proxy.new(DelayedModel, self, options)
|
27
29
|
end
|
28
|
-
|
29
|
-
|
30
|
+
|
31
|
+
def sidekiq_delay_for(interval, options = {})
|
32
|
+
Proxy.new(DelayedModel, self, options.merge("at" => Time.now.to_f + interval.to_f))
|
30
33
|
end
|
31
|
-
|
32
|
-
|
34
|
+
|
35
|
+
def sidekiq_delay_until(timestamp, options = {})
|
36
|
+
Proxy.new(DelayedModel, self, options.merge("at" => timestamp.to_f))
|
33
37
|
end
|
34
38
|
alias_method :delay, :sidekiq_delay
|
35
39
|
alias_method :delay_for, :sidekiq_delay_for
|
36
40
|
alias_method :delay_until, :sidekiq_delay_until
|
37
41
|
end
|
38
|
-
|
39
42
|
end
|
40
43
|
end
|
@@ -1,14 +1,16 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
|
-
|
2
|
+
|
3
|
+
require "sidekiq/extensions/generic_proxy"
|
3
4
|
|
4
5
|
module Sidekiq
|
5
6
|
module Extensions
|
6
7
|
##
|
7
|
-
# Adds
|
8
|
-
# execution to Sidekiq.
|
8
|
+
# Adds `delay`, `delay_for` and `delay_until` methods to all Classes to offload class method
|
9
|
+
# execution to Sidekiq.
|
9
10
|
#
|
10
|
-
#
|
11
|
-
#
|
11
|
+
# @example
|
12
|
+
# User.delay.delete_inactive
|
13
|
+
# Wikipedia.delay.download_changes_for(Date.today)
|
12
14
|
#
|
13
15
|
class DelayedClass
|
14
16
|
include Sidekiq::Worker
|
@@ -20,20 +22,21 @@ module Sidekiq
|
|
20
22
|
end
|
21
23
|
|
22
24
|
module Klass
|
23
|
-
def sidekiq_delay(options={})
|
25
|
+
def sidekiq_delay(options = {})
|
24
26
|
Proxy.new(DelayedClass, self, options)
|
25
27
|
end
|
26
|
-
|
27
|
-
|
28
|
+
|
29
|
+
def sidekiq_delay_for(interval, options = {})
|
30
|
+
Proxy.new(DelayedClass, self, options.merge("at" => Time.now.to_f + interval.to_f))
|
28
31
|
end
|
29
|
-
|
30
|
-
|
32
|
+
|
33
|
+
def sidekiq_delay_until(timestamp, options = {})
|
34
|
+
Proxy.new(DelayedClass, self, options.merge("at" => timestamp.to_f))
|
31
35
|
end
|
32
36
|
alias_method :delay, :sidekiq_delay
|
33
37
|
alias_method :delay_for, :sidekiq_delay_for
|
34
38
|
alias_method :delay_until, :sidekiq_delay_until
|
35
39
|
end
|
36
|
-
|
37
40
|
end
|
38
41
|
end
|
39
42
|
|
@@ -1,10 +1,13 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
|
-
|
2
|
+
|
3
|
+
require "yaml"
|
3
4
|
|
4
5
|
module Sidekiq
|
5
6
|
module Extensions
|
7
|
+
SIZE_LIMIT = 8_192
|
8
|
+
|
6
9
|
class Proxy < BasicObject
|
7
|
-
def initialize(performable, target, options={})
|
10
|
+
def initialize(performable, target, options = {})
|
8
11
|
@performable = performable
|
9
12
|
@target = target
|
10
13
|
@opts = options
|
@@ -17,9 +20,14 @@ module Sidekiq
|
|
17
20
|
# to JSON and then deserialized on the other side back into a
|
18
21
|
# Ruby object.
|
19
22
|
obj = [@target, name, args]
|
20
|
-
|
23
|
+
marshalled = ::YAML.dump(obj)
|
24
|
+
if marshalled.size > SIZE_LIMIT
|
25
|
+
::Sidekiq.logger.warn { "#{@target}.#{name} job argument is #{marshalled.bytesize} bytes, you should refactor it to reduce the size" }
|
26
|
+
end
|
27
|
+
@performable.client_push({"class" => @performable,
|
28
|
+
"args" => [marshalled],
|
29
|
+
"display_class" => "#{@target}.#{name}"}.merge(@opts))
|
21
30
|
end
|
22
31
|
end
|
23
|
-
|
24
32
|
end
|
25
33
|
end
|
data/lib/sidekiq/fetch.rb
CHANGED
@@ -1,5 +1,6 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
|
-
|
2
|
+
|
3
|
+
require "sidekiq"
|
3
4
|
|
4
5
|
module Sidekiq
|
5
6
|
class BasicFetch
|
@@ -7,68 +8,60 @@ module Sidekiq
|
|
7
8
|
# can check if the process is shutting down.
|
8
9
|
TIMEOUT = 2
|
9
10
|
|
10
|
-
UnitOfWork = Struct.new(:queue, :job)
|
11
|
+
UnitOfWork = Struct.new(:queue, :job) {
|
11
12
|
def acknowledge
|
12
13
|
# nothing to do
|
13
14
|
end
|
14
15
|
|
15
16
|
def queue_name
|
16
|
-
queue.
|
17
|
+
queue.delete_prefix("queue:")
|
17
18
|
end
|
18
19
|
|
19
20
|
def requeue
|
20
21
|
Sidekiq.redis do |conn|
|
21
|
-
conn.rpush(
|
22
|
+
conn.rpush(queue, job)
|
22
23
|
end
|
23
24
|
end
|
24
|
-
|
25
|
+
}
|
25
26
|
|
26
27
|
def initialize(options)
|
27
|
-
|
28
|
-
@
|
28
|
+
raise ArgumentError, "missing queue list" unless options[:queues]
|
29
|
+
@options = options
|
30
|
+
@strictly_ordered_queues = !!@options[:strict]
|
31
|
+
@queues = @options[:queues].map { |q| "queue:#{q}" }
|
29
32
|
if @strictly_ordered_queues
|
30
|
-
@queues
|
33
|
+
@queues.uniq!
|
31
34
|
@queues << TIMEOUT
|
32
35
|
end
|
33
36
|
end
|
34
37
|
|
35
38
|
def retrieve_work
|
36
|
-
|
37
|
-
|
38
|
-
|
39
|
-
|
40
|
-
|
41
|
-
|
42
|
-
# data from the first queue that has pending elements. We
|
43
|
-
# recreate the queue command each time we invoke Redis#brpop
|
44
|
-
# to honor weights and avoid queue starvation.
|
45
|
-
def queues_cmd
|
46
|
-
if @strictly_ordered_queues
|
47
|
-
@queues
|
48
|
-
else
|
49
|
-
queues = @queues.shuffle.uniq
|
50
|
-
queues << TIMEOUT
|
51
|
-
queues
|
39
|
+
qs = queues_cmd
|
40
|
+
# 4825 Sidekiq Pro with all queues paused will return an
|
41
|
+
# empty set of queues with a trailing TIMEOUT value.
|
42
|
+
if qs.size <= 1
|
43
|
+
sleep(TIMEOUT)
|
44
|
+
return nil
|
52
45
|
end
|
53
|
-
end
|
54
46
|
|
47
|
+
work = Sidekiq.redis { |conn| conn.brpop(*qs) }
|
48
|
+
UnitOfWork.new(*work) if work
|
49
|
+
end
|
55
50
|
|
56
|
-
|
57
|
-
# an instance method will make it async to the Fetcher actor
|
58
|
-
def self.bulk_requeue(inprogress, options)
|
51
|
+
def bulk_requeue(inprogress, options)
|
59
52
|
return if inprogress.empty?
|
60
53
|
|
61
54
|
Sidekiq.logger.debug { "Re-queueing terminated jobs" }
|
62
55
|
jobs_to_requeue = {}
|
63
56
|
inprogress.each do |unit_of_work|
|
64
|
-
jobs_to_requeue[unit_of_work.
|
65
|
-
jobs_to_requeue[unit_of_work.
|
57
|
+
jobs_to_requeue[unit_of_work.queue] ||= []
|
58
|
+
jobs_to_requeue[unit_of_work.queue] << unit_of_work.job
|
66
59
|
end
|
67
60
|
|
68
61
|
Sidekiq.redis do |conn|
|
69
62
|
conn.pipelined do
|
70
63
|
jobs_to_requeue.each do |queue, jobs|
|
71
|
-
conn.rpush(
|
64
|
+
conn.rpush(queue, jobs)
|
72
65
|
end
|
73
66
|
end
|
74
67
|
end
|
@@ -77,5 +70,20 @@ module Sidekiq
|
|
77
70
|
Sidekiq.logger.warn("Failed to requeue #{inprogress.size} jobs: #{ex.message}")
|
78
71
|
end
|
79
72
|
|
73
|
+
# Creating the Redis#brpop command takes into account any
|
74
|
+
# configured queue weights. By default Redis#brpop returns
|
75
|
+
# data from the first queue that has pending elements. We
|
76
|
+
# recreate the queue command each time we invoke Redis#brpop
|
77
|
+
# to honor weights and avoid queue starvation.
|
78
|
+
def queues_cmd
|
79
|
+
if @strictly_ordered_queues
|
80
|
+
@queues
|
81
|
+
else
|
82
|
+
permute = @queues.shuffle
|
83
|
+
permute.uniq!
|
84
|
+
permute << TIMEOUT
|
85
|
+
permute
|
86
|
+
end
|
87
|
+
end
|
80
88
|
end
|
81
89
|
end
|