sidekiq 5.2.10 → 7.2.0
Sign up to get free protection for your applications and to get access to all the features.
Potentially problematic release.
This version of sidekiq might be problematic. Click here for more details.
- checksums.yaml +4 -4
- data/Changes.md +600 -8
- data/LICENSE.txt +9 -0
- data/README.md +47 -50
- data/bin/sidekiq +22 -3
- data/bin/sidekiqload +213 -115
- data/bin/sidekiqmon +11 -0
- data/lib/generators/sidekiq/job_generator.rb +57 -0
- data/lib/generators/sidekiq/templates/{worker.rb.erb → job.rb.erb} +2 -2
- data/lib/generators/sidekiq/templates/{worker_spec.rb.erb → job_spec.rb.erb} +1 -1
- data/lib/generators/sidekiq/templates/{worker_test.rb.erb → job_test.rb.erb} +1 -1
- data/lib/sidekiq/api.rb +557 -354
- data/lib/sidekiq/capsule.rb +127 -0
- data/lib/sidekiq/cli.rb +204 -226
- data/lib/sidekiq/client.rb +127 -102
- data/lib/sidekiq/component.rb +68 -0
- data/lib/sidekiq/config.rb +287 -0
- data/lib/sidekiq/deploy.rb +62 -0
- data/lib/sidekiq/embedded.rb +61 -0
- data/lib/sidekiq/fetch.rb +49 -42
- data/lib/sidekiq/job.rb +374 -0
- data/lib/sidekiq/job_logger.rb +33 -7
- data/lib/sidekiq/job_retry.rb +147 -108
- data/lib/sidekiq/job_util.rb +107 -0
- data/lib/sidekiq/launcher.rb +203 -105
- data/lib/sidekiq/logger.rb +131 -0
- data/lib/sidekiq/manager.rb +43 -46
- data/lib/sidekiq/metrics/query.rb +155 -0
- data/lib/sidekiq/metrics/shared.rb +95 -0
- data/lib/sidekiq/metrics/tracking.rb +136 -0
- data/lib/sidekiq/middleware/chain.rb +113 -56
- data/lib/sidekiq/middleware/current_attributes.rb +95 -0
- data/lib/sidekiq/middleware/i18n.rb +7 -7
- data/lib/sidekiq/middleware/modules.rb +21 -0
- data/lib/sidekiq/monitor.rb +146 -0
- data/lib/sidekiq/paginator.rb +28 -16
- data/lib/sidekiq/processor.rb +122 -120
- data/lib/sidekiq/rails.rb +48 -38
- data/lib/sidekiq/redis_client_adapter.rb +111 -0
- data/lib/sidekiq/redis_connection.rb +39 -107
- data/lib/sidekiq/ring_buffer.rb +29 -0
- data/lib/sidekiq/scheduled.rb +111 -49
- data/lib/sidekiq/sd_notify.rb +149 -0
- data/lib/sidekiq/systemd.rb +24 -0
- data/lib/sidekiq/testing/inline.rb +6 -5
- data/lib/sidekiq/testing.rb +90 -89
- data/lib/sidekiq/transaction_aware_client.rb +44 -0
- data/lib/sidekiq/version.rb +3 -1
- data/lib/sidekiq/web/action.rb +15 -11
- data/lib/sidekiq/web/application.rb +186 -79
- data/lib/sidekiq/web/csrf_protection.rb +180 -0
- data/lib/sidekiq/web/helpers.rb +154 -115
- data/lib/sidekiq/web/router.rb +23 -19
- data/lib/sidekiq/web.rb +68 -107
- data/lib/sidekiq/worker_compatibility_alias.rb +13 -0
- data/lib/sidekiq.rb +92 -182
- data/sidekiq.gemspec +25 -16
- data/web/assets/images/apple-touch-icon.png +0 -0
- data/web/assets/javascripts/application.js +146 -61
- data/web/assets/javascripts/base-charts.js +106 -0
- data/web/assets/javascripts/chart.min.js +13 -0
- data/web/assets/javascripts/chartjs-plugin-annotation.min.js +7 -0
- data/web/assets/javascripts/dashboard-charts.js +182 -0
- data/web/assets/javascripts/dashboard.js +35 -293
- data/web/assets/javascripts/metrics.js +298 -0
- data/web/assets/stylesheets/application-dark.css +147 -0
- data/web/assets/stylesheets/application-rtl.css +2 -95
- data/web/assets/stylesheets/application.css +111 -522
- data/web/locales/ar.yml +71 -65
- data/web/locales/cs.yml +62 -62
- data/web/locales/da.yml +60 -53
- data/web/locales/de.yml +65 -53
- data/web/locales/el.yml +43 -24
- data/web/locales/en.yml +86 -66
- data/web/locales/es.yml +70 -54
- data/web/locales/fa.yml +65 -65
- data/web/locales/fr.yml +83 -62
- data/web/locales/gd.yml +99 -0
- data/web/locales/he.yml +65 -64
- data/web/locales/hi.yml +59 -59
- data/web/locales/it.yml +53 -53
- data/web/locales/ja.yml +75 -64
- data/web/locales/ko.yml +52 -52
- data/web/locales/lt.yml +83 -0
- data/web/locales/nb.yml +61 -61
- data/web/locales/nl.yml +52 -52
- data/web/locales/pl.yml +45 -45
- data/web/locales/pt-br.yml +83 -55
- data/web/locales/pt.yml +51 -51
- data/web/locales/ru.yml +68 -63
- data/web/locales/sv.yml +53 -53
- data/web/locales/ta.yml +60 -60
- data/web/locales/uk.yml +62 -61
- data/web/locales/ur.yml +64 -64
- data/web/locales/vi.yml +83 -0
- data/web/locales/zh-cn.yml +43 -16
- data/web/locales/zh-tw.yml +42 -8
- data/web/views/_footer.erb +6 -3
- data/web/views/_job_info.erb +21 -4
- data/web/views/_metrics_period_select.erb +12 -0
- data/web/views/_nav.erb +1 -1
- data/web/views/_paging.erb +2 -0
- data/web/views/_poll_link.erb +3 -6
- data/web/views/_summary.erb +7 -7
- data/web/views/busy.erb +77 -27
- data/web/views/dashboard.erb +48 -18
- data/web/views/dead.erb +3 -3
- data/web/views/filtering.erb +7 -0
- data/web/views/layout.erb +3 -1
- data/web/views/metrics.erb +91 -0
- data/web/views/metrics_for_job.erb +59 -0
- data/web/views/morgue.erb +14 -15
- data/web/views/queue.erb +33 -24
- data/web/views/queues.erb +19 -5
- data/web/views/retries.erb +16 -17
- data/web/views/retry.erb +3 -3
- data/web/views/scheduled.erb +17 -15
- metadata +71 -71
- data/.circleci/config.yml +0 -61
- data/.github/contributing.md +0 -32
- data/.github/issue_template.md +0 -11
- data/.gitignore +0 -15
- data/.travis.yml +0 -11
- data/3.0-Upgrade.md +0 -70
- data/4.0-Upgrade.md +0 -53
- data/5.0-Upgrade.md +0 -56
- data/COMM-LICENSE +0 -97
- data/Ent-Changes.md +0 -238
- data/Gemfile +0 -19
- data/LICENSE +0 -9
- data/Pro-2.0-Upgrade.md +0 -138
- data/Pro-3.0-Upgrade.md +0 -44
- data/Pro-4.0-Upgrade.md +0 -35
- data/Pro-Changes.md +0 -759
- data/Rakefile +0 -9
- data/bin/sidekiqctl +0 -20
- data/code_of_conduct.md +0 -50
- data/lib/generators/sidekiq/worker_generator.rb +0 -49
- data/lib/sidekiq/core_ext.rb +0 -1
- data/lib/sidekiq/ctl.rb +0 -221
- data/lib/sidekiq/delay.rb +0 -42
- data/lib/sidekiq/exception_handler.rb +0 -29
- data/lib/sidekiq/extensions/action_mailer.rb +0 -57
- data/lib/sidekiq/extensions/active_record.rb +0 -40
- data/lib/sidekiq/extensions/class_methods.rb +0 -40
- data/lib/sidekiq/extensions/generic_proxy.rb +0 -31
- data/lib/sidekiq/logging.rb +0 -122
- data/lib/sidekiq/middleware/server/active_record.rb +0 -23
- data/lib/sidekiq/util.rb +0 -66
- data/lib/sidekiq/worker.rb +0 -220
data/lib/sidekiq/job_retry.rb
CHANGED
@@ -1,6 +1,8 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
|
-
|
3
|
-
require
|
2
|
+
|
3
|
+
require "zlib"
|
4
|
+
require "base64"
|
5
|
+
require "sidekiq/component"
|
4
6
|
|
5
7
|
module Sidekiq
|
6
8
|
##
|
@@ -21,18 +23,19 @@ module Sidekiq
|
|
21
23
|
#
|
22
24
|
# A job looks like:
|
23
25
|
#
|
24
|
-
# { 'class' => '
|
26
|
+
# { 'class' => 'HardJob', 'args' => [1, 2, 'foo'], 'retry' => true }
|
25
27
|
#
|
26
28
|
# The 'retry' option also accepts a number (in place of 'true'):
|
27
29
|
#
|
28
|
-
# { 'class' => '
|
30
|
+
# { 'class' => 'HardJob', 'args' => [1, 2, 'foo'], 'retry' => 5 }
|
29
31
|
#
|
30
32
|
# The job will be retried this number of times before giving up. (If simply
|
31
33
|
# 'true', Sidekiq retries 25 times)
|
32
34
|
#
|
33
|
-
#
|
35
|
+
# Relevant options for job retries:
|
34
36
|
#
|
35
|
-
# * 'queue' - the queue
|
37
|
+
# * 'queue' - the queue for the initial job
|
38
|
+
# * 'retry_queue' - if job retries should be pushed to a different (e.g. lower priority) queue
|
36
39
|
# * 'retry_count' - number of times we've retried so far.
|
37
40
|
# * 'error_message' - the message from the exception
|
38
41
|
# * 'error_class' - the exception class
|
@@ -46,31 +49,35 @@ module Sidekiq
|
|
46
49
|
# The default number of retries is 25 which works out to about 3 weeks
|
47
50
|
# You can change the default maximum number of retries in your initializer:
|
48
51
|
#
|
49
|
-
# Sidekiq.
|
52
|
+
# Sidekiq.default_configuration[:max_retries] = 7
|
50
53
|
#
|
51
|
-
# or limit the number of retries for a particular
|
54
|
+
# or limit the number of retries for a particular job and send retries to
|
55
|
+
# a low priority queue with:
|
52
56
|
#
|
53
|
-
# class
|
54
|
-
# include Sidekiq::
|
55
|
-
# sidekiq_options :
|
57
|
+
# class MyJob
|
58
|
+
# include Sidekiq::Job
|
59
|
+
# sidekiq_options retry: 10, retry_queue: 'low'
|
56
60
|
# end
|
57
61
|
#
|
58
62
|
class JobRetry
|
59
63
|
class Handled < ::RuntimeError; end
|
64
|
+
|
60
65
|
class Skip < Handled; end
|
61
66
|
|
62
|
-
include Sidekiq::
|
67
|
+
include Sidekiq::Component
|
63
68
|
|
64
69
|
DEFAULT_MAX_RETRY_ATTEMPTS = 25
|
65
70
|
|
66
|
-
def initialize(
|
67
|
-
@
|
71
|
+
def initialize(capsule)
|
72
|
+
@config = @capsule = capsule
|
73
|
+
@max_retries = Sidekiq.default_configuration[:max_retries] || DEFAULT_MAX_RETRY_ATTEMPTS
|
74
|
+
@backtrace_cleaner = Sidekiq.default_configuration[:backtrace_cleaner]
|
68
75
|
end
|
69
76
|
|
70
77
|
# The global retry handler requires only the barest of data.
|
71
78
|
# We want to be able to retry as much as possible so we don't
|
72
|
-
# require the
|
73
|
-
def global(
|
79
|
+
# require the job to be instantiated.
|
80
|
+
def global(jobstr, queue)
|
74
81
|
yield
|
75
82
|
rescue Handled => ex
|
76
83
|
raise ex
|
@@ -81,31 +88,29 @@ module Sidekiq
|
|
81
88
|
# ignore, will be pushed back onto queue during hard_shutdown
|
82
89
|
raise Sidekiq::Shutdown if exception_caused_by_shutdown?(e)
|
83
90
|
|
84
|
-
|
85
|
-
|
91
|
+
msg = Sidekiq.load_json(jobstr)
|
92
|
+
if msg["retry"]
|
93
|
+
process_retry(nil, msg, queue, e)
|
86
94
|
else
|
87
|
-
|
88
|
-
|
89
|
-
|
90
|
-
|
91
|
-
handle_exception(handler_ex, { context: "Error calling death handler", job: msg })
|
92
|
-
end
|
95
|
+
@capsule.config.death_handlers.each do |handler|
|
96
|
+
handler.call(msg, e)
|
97
|
+
rescue => handler_ex
|
98
|
+
handle_exception(handler_ex, {context: "Error calling death handler", job: msg})
|
93
99
|
end
|
94
100
|
end
|
95
101
|
|
96
102
|
raise Handled
|
97
103
|
end
|
98
104
|
|
99
|
-
|
100
105
|
# The local retry support means that any errors that occur within
|
101
|
-
# this block can be associated with the given
|
106
|
+
# this block can be associated with the given job instance.
|
102
107
|
# This is required to support the `sidekiq_retries_exhausted` block.
|
103
108
|
#
|
104
109
|
# Note that any exception from the block is wrapped in the Skip
|
105
110
|
# exception so the global block does not reprocess the error. The
|
106
111
|
# Skip exception is unwrapped within Sidekiq::Processor#process before
|
107
112
|
# calling the handle_exception handlers.
|
108
|
-
def local(
|
113
|
+
def local(jobinst, jobstr, queue)
|
109
114
|
yield
|
110
115
|
rescue Handled => ex
|
111
116
|
raise ex
|
@@ -116,12 +121,13 @@ module Sidekiq
|
|
116
121
|
# ignore, will be pushed back onto queue during hard_shutdown
|
117
122
|
raise Sidekiq::Shutdown if exception_caused_by_shutdown?(e)
|
118
123
|
|
119
|
-
|
120
|
-
|
124
|
+
msg = Sidekiq.load_json(jobstr)
|
125
|
+
if msg["retry"].nil?
|
126
|
+
msg["retry"] = jobinst.class.get_sidekiq_options["retry"]
|
121
127
|
end
|
122
128
|
|
123
|
-
raise e unless msg[
|
124
|
-
|
129
|
+
raise e unless msg["retry"]
|
130
|
+
process_retry(jobinst, msg, queue, e)
|
125
131
|
# We've handled this error associated with this job, don't
|
126
132
|
# need to handle it at the global level
|
127
133
|
raise Skip
|
@@ -129,17 +135,13 @@ module Sidekiq
|
|
129
135
|
|
130
136
|
private
|
131
137
|
|
132
|
-
# Note that +
|
133
|
-
# instantiate the
|
138
|
+
# Note that +jobinst+ can be nil here if an error is raised before we can
|
139
|
+
# instantiate the job instance. All access must be guarded and
|
134
140
|
# best effort.
|
135
|
-
def
|
136
|
-
max_retry_attempts = retry_attempts_from(msg[
|
141
|
+
def process_retry(jobinst, msg, queue, exception)
|
142
|
+
max_retry_attempts = retry_attempts_from(msg["retry"], @max_retries)
|
137
143
|
|
138
|
-
msg[
|
139
|
-
msg['retry_queue']
|
140
|
-
else
|
141
|
-
queue
|
142
|
-
end
|
144
|
+
msg["queue"] = (msg["retry_queue"] || queue)
|
143
145
|
|
144
146
|
m = exception_message(exception)
|
145
147
|
if m.respond_to?(:scrub!)
|
@@ -147,62 +149,118 @@ module Sidekiq
|
|
147
149
|
m.scrub!
|
148
150
|
end
|
149
151
|
|
150
|
-
msg[
|
151
|
-
msg[
|
152
|
-
count = if msg[
|
153
|
-
msg[
|
154
|
-
msg[
|
152
|
+
msg["error_message"] = m
|
153
|
+
msg["error_class"] = exception.class.name
|
154
|
+
count = if msg["retry_count"]
|
155
|
+
msg["retried_at"] = Time.now.to_f
|
156
|
+
msg["retry_count"] += 1
|
155
157
|
else
|
156
|
-
msg[
|
157
|
-
msg[
|
158
|
+
msg["failed_at"] = Time.now.to_f
|
159
|
+
msg["retry_count"] = 0
|
158
160
|
end
|
159
161
|
|
160
|
-
if msg[
|
161
|
-
|
162
|
-
|
163
|
-
|
164
|
-
|
165
|
-
|
162
|
+
if msg["backtrace"]
|
163
|
+
backtrace = @backtrace_cleaner.call(exception.backtrace)
|
164
|
+
lines = if msg["backtrace"] == true
|
165
|
+
backtrace
|
166
|
+
else
|
167
|
+
backtrace[0...msg["backtrace"].to_i]
|
168
|
+
end
|
169
|
+
|
170
|
+
msg["error_backtrace"] = compress_backtrace(lines)
|
166
171
|
end
|
167
172
|
|
168
|
-
if count
|
169
|
-
|
170
|
-
|
171
|
-
|
172
|
-
|
173
|
-
|
174
|
-
|
175
|
-
|
176
|
-
|
177
|
-
|
178
|
-
|
179
|
-
|
173
|
+
return retries_exhausted(jobinst, msg, exception) if count >= max_retry_attempts
|
174
|
+
|
175
|
+
rf = msg["retry_for"]
|
176
|
+
return retries_exhausted(jobinst, msg, exception) if rf && ((msg["failed_at"] + rf) < Time.now.to_f)
|
177
|
+
|
178
|
+
strategy, delay = delay_for(jobinst, count, exception, msg)
|
179
|
+
case strategy
|
180
|
+
when :discard
|
181
|
+
return # poof!
|
182
|
+
when :kill
|
183
|
+
return retries_exhausted(jobinst, msg, exception)
|
184
|
+
end
|
185
|
+
|
186
|
+
# Logging here can break retries if the logging device raises ENOSPC #3979
|
187
|
+
# logger.debug { "Failure! Retry #{count} in #{delay} seconds" }
|
188
|
+
jitter = rand(10) * (count + 1)
|
189
|
+
retry_at = Time.now.to_f + delay + jitter
|
190
|
+
payload = Sidekiq.dump_json(msg)
|
191
|
+
redis do |conn|
|
192
|
+
conn.zadd("retry", retry_at.to_s, payload)
|
180
193
|
end
|
181
194
|
end
|
182
195
|
|
183
|
-
|
184
|
-
|
185
|
-
|
186
|
-
|
187
|
-
|
188
|
-
|
196
|
+
# returns (strategy, seconds)
|
197
|
+
def delay_for(jobinst, count, exception, msg)
|
198
|
+
rv = begin
|
199
|
+
# sidekiq_retry_in can return two different things:
|
200
|
+
# 1. When to retry next, as an integer of seconds
|
201
|
+
# 2. A symbol which re-routes the job elsewhere, e.g. :discard, :kill, :default
|
202
|
+
block = jobinst&.sidekiq_retry_in_block
|
203
|
+
|
204
|
+
# the sidekiq_retry_in_block can be defined in a wrapped class (ActiveJob for instance)
|
205
|
+
unless msg["wrapped"].nil?
|
206
|
+
wrapped = Object.const_get(msg["wrapped"])
|
207
|
+
block = wrapped.respond_to?(:sidekiq_retry_in_block) ? wrapped.sidekiq_retry_in_block : nil
|
208
|
+
end
|
209
|
+
block&.call(count, exception, msg)
|
210
|
+
rescue Exception => e
|
211
|
+
handle_exception(e, {context: "Failure scheduling retry using the defined `sidekiq_retry_in` in #{jobinst.class.name}, falling back to default"})
|
212
|
+
nil
|
189
213
|
end
|
190
214
|
|
191
|
-
|
192
|
-
|
193
|
-
|
194
|
-
|
195
|
-
|
215
|
+
rv = rv.to_i if rv.respond_to?(:to_i)
|
216
|
+
delay = (count**4) + 15
|
217
|
+
if Integer === rv && rv > 0
|
218
|
+
delay = rv
|
219
|
+
elsif rv == :discard
|
220
|
+
return [:discard, nil] # do nothing, job goes poof
|
221
|
+
elsif rv == :kill
|
222
|
+
return [:kill, nil]
|
223
|
+
end
|
224
|
+
|
225
|
+
[:default, delay]
|
226
|
+
end
|
227
|
+
|
228
|
+
def retries_exhausted(jobinst, msg, exception)
|
229
|
+
rv = begin
|
230
|
+
block = jobinst&.sidekiq_retries_exhausted_block
|
231
|
+
|
232
|
+
# the sidekiq_retries_exhausted_block can be defined in a wrapped class (ActiveJob for instance)
|
233
|
+
unless msg["wrapped"].nil?
|
234
|
+
wrapped = Object.const_get(msg["wrapped"])
|
235
|
+
block = wrapped.respond_to?(:sidekiq_retries_exhausted_block) ? wrapped.sidekiq_retries_exhausted_block : nil
|
196
236
|
end
|
237
|
+
block&.call(msg, exception)
|
238
|
+
rescue => e
|
239
|
+
handle_exception(e, {context: "Error calling retries_exhausted", job: msg})
|
197
240
|
end
|
198
241
|
|
199
|
-
|
242
|
+
return if rv == :discard # poof!
|
243
|
+
send_to_morgue(msg) unless msg["dead"] == false
|
244
|
+
|
245
|
+
@capsule.config.death_handlers.each do |handler|
|
246
|
+
handler.call(msg, exception)
|
247
|
+
rescue => e
|
248
|
+
handle_exception(e, {context: "Error calling death handler", job: msg})
|
249
|
+
end
|
200
250
|
end
|
201
251
|
|
202
252
|
def send_to_morgue(msg)
|
203
|
-
logger.info { "Adding dead #{msg[
|
253
|
+
logger.info { "Adding dead #{msg["class"]} job #{msg["jid"]}" }
|
204
254
|
payload = Sidekiq.dump_json(msg)
|
205
|
-
|
255
|
+
now = Time.now.to_f
|
256
|
+
|
257
|
+
redis do |conn|
|
258
|
+
conn.multi do |xa|
|
259
|
+
xa.zadd("dead", now.to_s, payload)
|
260
|
+
xa.zremrangebyscore("dead", "-inf", now - @capsule.config[:dead_timeout_in_seconds])
|
261
|
+
xa.zremrangebyrank("dead", 0, - @capsule.config[:dead_max_jobs])
|
262
|
+
end
|
263
|
+
end
|
206
264
|
end
|
207
265
|
|
208
266
|
def retry_attempts_from(msg_retry, default)
|
@@ -213,28 +271,6 @@ module Sidekiq
|
|
213
271
|
end
|
214
272
|
end
|
215
273
|
|
216
|
-
def delay_for(worker, count, exception)
|
217
|
-
if worker && worker.sidekiq_retry_in_block
|
218
|
-
custom_retry_in = retry_in(worker, count, exception).to_i
|
219
|
-
return custom_retry_in if custom_retry_in > 0
|
220
|
-
end
|
221
|
-
seconds_to_delay(count)
|
222
|
-
end
|
223
|
-
|
224
|
-
# delayed_job uses the same basic formula
|
225
|
-
def seconds_to_delay(count)
|
226
|
-
(count ** 4) + 15 + (rand(30)*(count+1))
|
227
|
-
end
|
228
|
-
|
229
|
-
def retry_in(worker, count, exception)
|
230
|
-
begin
|
231
|
-
worker.sidekiq_retry_in_block.call(count, exception)
|
232
|
-
rescue Exception => e
|
233
|
-
handle_exception(e, { context: "Failure scheduling retry using the defined `sidekiq_retry_in` in #{worker.class.name}, falling back to default" })
|
234
|
-
nil
|
235
|
-
end
|
236
|
-
end
|
237
|
-
|
238
274
|
def exception_caused_by_shutdown?(e, checked_causes = [])
|
239
275
|
return false unless e.cause
|
240
276
|
|
@@ -249,14 +285,17 @@ module Sidekiq
|
|
249
285
|
# Extract message from exception.
|
250
286
|
# Set a default if the message raises an error
|
251
287
|
def exception_message(exception)
|
252
|
-
|
253
|
-
|
254
|
-
|
255
|
-
|
256
|
-
|
257
|
-
"!!! ERROR MESSAGE THREW AN ERROR !!!".dup
|
258
|
-
end
|
288
|
+
# App code can stuff all sorts of crazy binary data into the error message
|
289
|
+
# that won't convert to JSON.
|
290
|
+
exception.message.to_s[0, 10_000]
|
291
|
+
rescue
|
292
|
+
+"!!! ERROR MESSAGE THREW AN ERROR !!!"
|
259
293
|
end
|
260
294
|
|
295
|
+
def compress_backtrace(backtrace)
|
296
|
+
serialized = Sidekiq.dump_json(backtrace)
|
297
|
+
compressed = Zlib::Deflate.deflate(serialized)
|
298
|
+
Base64.encode64(compressed)
|
299
|
+
end
|
261
300
|
end
|
262
301
|
end
|
@@ -0,0 +1,107 @@
|
|
1
|
+
require "securerandom"
|
2
|
+
require "time"
|
3
|
+
|
4
|
+
module Sidekiq
|
5
|
+
module JobUtil
|
6
|
+
# These functions encapsulate various job utilities.
|
7
|
+
|
8
|
+
TRANSIENT_ATTRIBUTES = %w[]
|
9
|
+
|
10
|
+
def validate(item)
|
11
|
+
raise(ArgumentError, "Job must be a Hash with 'class' and 'args' keys: `#{item}`") unless item.is_a?(Hash) && item.key?("class") && item.key?("args")
|
12
|
+
raise(ArgumentError, "Job args must be an Array: `#{item}`") unless item["args"].is_a?(Array) || item["args"].is_a?(Enumerator::Lazy)
|
13
|
+
raise(ArgumentError, "Job class must be either a Class or String representation of the class name: `#{item}`") unless item["class"].is_a?(Class) || item["class"].is_a?(String)
|
14
|
+
raise(ArgumentError, "Job 'at' must be a Numeric timestamp: `#{item}`") if item.key?("at") && !item["at"].is_a?(Numeric)
|
15
|
+
raise(ArgumentError, "Job tags must be an Array: `#{item}`") if item["tags"] && !item["tags"].is_a?(Array)
|
16
|
+
raise(ArgumentError, "retry_for must be a relative amount of time, e.g. 48.hours `#{item}`") if item["retry_for"] && item["retry_for"] > 1_000_000_000
|
17
|
+
end
|
18
|
+
|
19
|
+
def verify_json(item)
|
20
|
+
job_class = item["wrapped"] || item["class"]
|
21
|
+
args = item["args"]
|
22
|
+
mode = Sidekiq::Config::DEFAULTS[:on_complex_arguments]
|
23
|
+
|
24
|
+
if mode == :raise || mode == :warn
|
25
|
+
if (unsafe_item = json_unsafe?(args))
|
26
|
+
msg = <<~EOM
|
27
|
+
Job arguments to #{job_class} must be native JSON types, but #{unsafe_item.inspect} is a #{unsafe_item.class}.
|
28
|
+
See https://github.com/sidekiq/sidekiq/wiki/Best-Practices
|
29
|
+
To disable this error, add `Sidekiq.strict_args!(false)` to your initializer.
|
30
|
+
EOM
|
31
|
+
|
32
|
+
if mode == :raise
|
33
|
+
raise(ArgumentError, msg)
|
34
|
+
else
|
35
|
+
warn(msg)
|
36
|
+
end
|
37
|
+
end
|
38
|
+
end
|
39
|
+
end
|
40
|
+
|
41
|
+
def normalize_item(item)
|
42
|
+
validate(item)
|
43
|
+
|
44
|
+
# merge in the default sidekiq_options for the item's class and/or wrapped element
|
45
|
+
# this allows ActiveJobs to control sidekiq_options too.
|
46
|
+
defaults = normalized_hash(item["class"])
|
47
|
+
defaults = defaults.merge(item["wrapped"].get_sidekiq_options) if item["wrapped"].respond_to?(:get_sidekiq_options)
|
48
|
+
item = defaults.merge(item)
|
49
|
+
|
50
|
+
raise(ArgumentError, "Job must include a valid queue name") if item["queue"].nil? || item["queue"] == ""
|
51
|
+
|
52
|
+
# remove job attributes which aren't necessary to persist into Redis
|
53
|
+
TRANSIENT_ATTRIBUTES.each { |key| item.delete(key) }
|
54
|
+
|
55
|
+
item["jid"] ||= SecureRandom.hex(12)
|
56
|
+
item["class"] = item["class"].to_s
|
57
|
+
item["queue"] = item["queue"].to_s
|
58
|
+
item["retry_for"] = item["retry_for"].to_i if item["retry_for"]
|
59
|
+
item["created_at"] ||= Time.now.to_f
|
60
|
+
item
|
61
|
+
end
|
62
|
+
|
63
|
+
def normalized_hash(item_class)
|
64
|
+
if item_class.is_a?(Class)
|
65
|
+
raise(ArgumentError, "Message must include a Sidekiq::Job class, not class name: #{item_class.ancestors.inspect}") unless item_class.respond_to?(:get_sidekiq_options)
|
66
|
+
item_class.get_sidekiq_options
|
67
|
+
else
|
68
|
+
Sidekiq.default_job_options
|
69
|
+
end
|
70
|
+
end
|
71
|
+
|
72
|
+
private
|
73
|
+
|
74
|
+
RECURSIVE_JSON_UNSAFE = {
|
75
|
+
Integer => ->(val) {},
|
76
|
+
Float => ->(val) {},
|
77
|
+
TrueClass => ->(val) {},
|
78
|
+
FalseClass => ->(val) {},
|
79
|
+
NilClass => ->(val) {},
|
80
|
+
String => ->(val) {},
|
81
|
+
Array => ->(val) {
|
82
|
+
val.each do |e|
|
83
|
+
unsafe_item = RECURSIVE_JSON_UNSAFE[e.class].call(e)
|
84
|
+
return unsafe_item unless unsafe_item.nil?
|
85
|
+
end
|
86
|
+
nil
|
87
|
+
},
|
88
|
+
Hash => ->(val) {
|
89
|
+
val.each do |k, v|
|
90
|
+
return k unless String === k
|
91
|
+
|
92
|
+
unsafe_item = RECURSIVE_JSON_UNSAFE[v.class].call(v)
|
93
|
+
return unsafe_item unless unsafe_item.nil?
|
94
|
+
end
|
95
|
+
nil
|
96
|
+
}
|
97
|
+
}
|
98
|
+
|
99
|
+
RECURSIVE_JSON_UNSAFE.default = ->(val) { val }
|
100
|
+
RECURSIVE_JSON_UNSAFE.compare_by_identity
|
101
|
+
private_constant :RECURSIVE_JSON_UNSAFE
|
102
|
+
|
103
|
+
def json_unsafe?(item)
|
104
|
+
RECURSIVE_JSON_UNSAFE[item.class].call(item)
|
105
|
+
end
|
106
|
+
end
|
107
|
+
end
|