sidekiq 6.4.1 → 7.0.7
Sign up to get free protection for your applications and to get access to all the features.
Potentially problematic release.
This version of sidekiq might be problematic. Click here for more details.
- checksums.yaml +4 -4
- data/Changes.md +172 -12
- data/README.md +41 -33
- data/bin/sidekiq +3 -8
- data/bin/sidekiqload +188 -114
- data/bin/sidekiqmon +3 -0
- data/lib/sidekiq/api.rb +275 -161
- data/lib/sidekiq/capsule.rb +127 -0
- data/lib/sidekiq/cli.rb +83 -88
- data/lib/sidekiq/client.rb +55 -43
- data/lib/sidekiq/component.rb +68 -0
- data/lib/sidekiq/config.rb +270 -0
- data/lib/sidekiq/deploy.rb +62 -0
- data/lib/sidekiq/embedded.rb +61 -0
- data/lib/sidekiq/fetch.rb +21 -22
- data/lib/sidekiq/job.rb +375 -10
- data/lib/sidekiq/job_logger.rb +2 -2
- data/lib/sidekiq/job_retry.rb +76 -54
- data/lib/sidekiq/job_util.rb +59 -19
- data/lib/sidekiq/launcher.rb +90 -82
- data/lib/sidekiq/logger.rb +6 -45
- data/lib/sidekiq/manager.rb +33 -32
- data/lib/sidekiq/metrics/query.rb +153 -0
- data/lib/sidekiq/metrics/shared.rb +95 -0
- data/lib/sidekiq/metrics/tracking.rb +136 -0
- data/lib/sidekiq/middleware/chain.rb +96 -51
- data/lib/sidekiq/middleware/current_attributes.rb +16 -17
- data/lib/sidekiq/middleware/i18n.rb +6 -4
- data/lib/sidekiq/middleware/modules.rb +21 -0
- data/lib/sidekiq/monitor.rb +17 -4
- data/lib/sidekiq/paginator.rb +11 -3
- data/lib/sidekiq/processor.rb +60 -60
- data/lib/sidekiq/rails.rb +12 -10
- data/lib/sidekiq/redis_client_adapter.rb +115 -0
- data/lib/sidekiq/redis_connection.rb +13 -82
- data/lib/sidekiq/ring_buffer.rb +29 -0
- data/lib/sidekiq/scheduled.rb +65 -37
- data/lib/sidekiq/testing/inline.rb +4 -4
- data/lib/sidekiq/testing.rb +41 -68
- data/lib/sidekiq/transaction_aware_client.rb +44 -0
- data/lib/sidekiq/version.rb +2 -1
- data/lib/sidekiq/web/action.rb +3 -3
- data/lib/sidekiq/web/application.rb +40 -9
- data/lib/sidekiq/web/csrf_protection.rb +3 -3
- data/lib/sidekiq/web/helpers.rb +34 -20
- data/lib/sidekiq/web.rb +7 -14
- data/lib/sidekiq/worker_compatibility_alias.rb +13 -0
- data/lib/sidekiq.rb +84 -207
- data/sidekiq.gemspec +20 -10
- data/web/assets/javascripts/application.js +76 -26
- data/web/assets/javascripts/base-charts.js +106 -0
- data/web/assets/javascripts/chart.min.js +13 -0
- data/web/assets/javascripts/chartjs-plugin-annotation.min.js +7 -0
- data/web/assets/javascripts/dashboard-charts.js +166 -0
- data/web/assets/javascripts/dashboard.js +3 -240
- data/web/assets/javascripts/metrics.js +264 -0
- data/web/assets/stylesheets/application-dark.css +4 -0
- data/web/assets/stylesheets/application-rtl.css +2 -91
- data/web/assets/stylesheets/application.css +66 -299
- data/web/locales/ar.yml +70 -70
- data/web/locales/cs.yml +62 -62
- data/web/locales/da.yml +60 -53
- data/web/locales/de.yml +65 -65
- data/web/locales/el.yml +43 -24
- data/web/locales/en.yml +82 -69
- data/web/locales/es.yml +68 -68
- data/web/locales/fa.yml +65 -65
- data/web/locales/fr.yml +67 -67
- data/web/locales/he.yml +65 -64
- data/web/locales/hi.yml +59 -59
- data/web/locales/it.yml +53 -53
- data/web/locales/ja.yml +73 -68
- data/web/locales/ko.yml +52 -52
- data/web/locales/lt.yml +66 -66
- data/web/locales/nb.yml +61 -61
- data/web/locales/nl.yml +52 -52
- data/web/locales/pl.yml +45 -45
- data/web/locales/pt-br.yml +63 -55
- data/web/locales/pt.yml +51 -51
- data/web/locales/ru.yml +67 -66
- data/web/locales/sv.yml +53 -53
- data/web/locales/ta.yml +60 -60
- data/web/locales/uk.yml +62 -61
- data/web/locales/ur.yml +64 -64
- data/web/locales/vi.yml +67 -67
- data/web/locales/zh-cn.yml +43 -16
- data/web/locales/zh-tw.yml +42 -8
- data/web/views/_footer.erb +5 -2
- data/web/views/_job_info.erb +18 -2
- data/web/views/_metrics_period_select.erb +12 -0
- data/web/views/_nav.erb +1 -1
- data/web/views/_paging.erb +2 -0
- data/web/views/_poll_link.erb +1 -1
- data/web/views/_summary.erb +1 -1
- data/web/views/busy.erb +42 -26
- data/web/views/dashboard.erb +36 -4
- data/web/views/metrics.erb +82 -0
- data/web/views/metrics_for_job.erb +71 -0
- data/web/views/morgue.erb +5 -9
- data/web/views/queue.erb +15 -15
- data/web/views/queues.erb +3 -1
- data/web/views/retries.erb +5 -9
- data/web/views/scheduled.erb +12 -13
- metadata +63 -28
- data/lib/sidekiq/delay.rb +0 -43
- data/lib/sidekiq/exception_handler.rb +0 -27
- data/lib/sidekiq/extensions/action_mailer.rb +0 -48
- data/lib/sidekiq/extensions/active_record.rb +0 -43
- data/lib/sidekiq/extensions/class_methods.rb +0 -43
- data/lib/sidekiq/extensions/generic_proxy.rb +0 -33
- data/lib/sidekiq/util.rb +0 -108
- data/lib/sidekiq/worker.rb +0 -362
- /data/{LICENSE → LICENSE.txt} +0 -0
data/lib/sidekiq/scheduled.rb
CHANGED
@@ -1,14 +1,15 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
3
|
require "sidekiq"
|
4
|
-
require "sidekiq/
|
5
|
-
require "sidekiq/api"
|
4
|
+
require "sidekiq/component"
|
6
5
|
|
7
6
|
module Sidekiq
|
8
7
|
module Scheduled
|
9
8
|
SETS = %w[retry schedule]
|
10
9
|
|
11
10
|
class Enq
|
11
|
+
include Sidekiq::Component
|
12
|
+
|
12
13
|
LUA_ZPOPBYSCORE = <<~LUA
|
13
14
|
local key, now = KEYS[1], ARGV[1]
|
14
15
|
local jobs = redis.call("zrangebyscore", key, "-inf", now, "limit", 0, 1)
|
@@ -18,7 +19,9 @@ module Sidekiq
|
|
18
19
|
end
|
19
20
|
LUA
|
20
21
|
|
21
|
-
def initialize
|
22
|
+
def initialize(container)
|
23
|
+
@config = container
|
24
|
+
@client = Sidekiq::Client.new(config: container)
|
22
25
|
@done = false
|
23
26
|
@lua_zpopbyscore_sha = nil
|
24
27
|
end
|
@@ -26,15 +29,15 @@ module Sidekiq
|
|
26
29
|
def enqueue_jobs(sorted_sets = SETS)
|
27
30
|
# A job's "score" in Redis is the time at which it should be processed.
|
28
31
|
# Just check Redis for the set of jobs with a timestamp before now.
|
29
|
-
|
32
|
+
redis do |conn|
|
30
33
|
sorted_sets.each do |sorted_set|
|
31
34
|
# Get next item in the queue with score (time to execute) <= now.
|
32
35
|
# We need to go through the list one at a time to reduce the risk of something
|
33
36
|
# going wrong between the time jobs are popped from the scheduled queue and when
|
34
37
|
# they are pushed onto a work queue and losing the jobs.
|
35
38
|
while !@done && (job = zpopbyscore(conn, keys: [sorted_set], argv: [Time.now.to_f.to_s]))
|
36
|
-
|
37
|
-
|
39
|
+
@client.push(Sidekiq.load_json(job))
|
40
|
+
logger.debug { "enqueued #{sorted_set}: #{job}" }
|
38
41
|
end
|
39
42
|
end
|
40
43
|
end
|
@@ -48,12 +51,11 @@ module Sidekiq
|
|
48
51
|
|
49
52
|
def zpopbyscore(conn, keys: nil, argv: nil)
|
50
53
|
if @lua_zpopbyscore_sha.nil?
|
51
|
-
|
52
|
-
@lua_zpopbyscore_sha = raw_conn.script(:load, LUA_ZPOPBYSCORE)
|
54
|
+
@lua_zpopbyscore_sha = conn.script(:load, LUA_ZPOPBYSCORE)
|
53
55
|
end
|
54
56
|
|
55
|
-
conn.
|
56
|
-
rescue
|
57
|
+
conn.call("EVALSHA", @lua_zpopbyscore_sha, keys.size, *keys, *argv)
|
58
|
+
rescue RedisClient::CommandError => e
|
57
59
|
raise unless e.message.start_with?("NOSCRIPT")
|
58
60
|
|
59
61
|
@lua_zpopbyscore_sha = nil
|
@@ -67,12 +69,13 @@ module Sidekiq
|
|
67
69
|
# just pops the job back onto its original queue so the
|
68
70
|
# workers can pick it up like any other job.
|
69
71
|
class Poller
|
70
|
-
include
|
72
|
+
include Sidekiq::Component
|
71
73
|
|
72
74
|
INITIAL_WAIT = 10
|
73
75
|
|
74
|
-
def initialize
|
75
|
-
@
|
76
|
+
def initialize(config)
|
77
|
+
@config = config
|
78
|
+
@enq = (config[:scheduled_enq] || Sidekiq::Scheduled::Enq).new(config)
|
76
79
|
@sleeper = ConnectionPool::TimedStack.new
|
77
80
|
@done = false
|
78
81
|
@thread = nil
|
@@ -82,14 +85,10 @@ module Sidekiq
|
|
82
85
|
# Shut down this instance, will pause until the thread is dead.
|
83
86
|
def terminate
|
84
87
|
@done = true
|
85
|
-
@enq.terminate
|
88
|
+
@enq.terminate
|
86
89
|
|
87
|
-
|
88
|
-
|
89
|
-
@thread = nil
|
90
|
-
@sleeper << 0
|
91
|
-
t.value
|
92
|
-
end
|
90
|
+
@sleeper << 0
|
91
|
+
@thread&.value
|
93
92
|
end
|
94
93
|
|
95
94
|
def start
|
@@ -100,7 +99,7 @@ module Sidekiq
|
|
100
99
|
enqueue
|
101
100
|
wait
|
102
101
|
end
|
103
|
-
|
102
|
+
logger.info("Scheduler exiting...")
|
104
103
|
}
|
105
104
|
end
|
106
105
|
|
@@ -147,13 +146,16 @@ module Sidekiq
|
|
147
146
|
# As we run more processes, the scheduling interval average will approach an even spread
|
148
147
|
# between 0 and poll interval so we don't need this artifical boost.
|
149
148
|
#
|
150
|
-
|
149
|
+
count = process_count
|
150
|
+
interval = poll_interval_average(count)
|
151
|
+
|
152
|
+
if count < 10
|
151
153
|
# For small clusters, calculate a random interval that is ±50% the desired average.
|
152
|
-
|
154
|
+
interval * rand + interval.to_f / 2
|
153
155
|
else
|
154
156
|
# With 10+ processes, we should have enough randomness to get decent polling
|
155
157
|
# across the entire timespan
|
156
|
-
|
158
|
+
interval * rand
|
157
159
|
end
|
158
160
|
end
|
159
161
|
|
@@ -170,38 +172,64 @@ module Sidekiq
|
|
170
172
|
# the same time: the thundering herd problem.
|
171
173
|
#
|
172
174
|
# We only do this if poll_interval_average is unset (the default).
|
173
|
-
def poll_interval_average
|
174
|
-
|
175
|
+
def poll_interval_average(count)
|
176
|
+
@config[:poll_interval_average] || scaled_poll_interval(count)
|
175
177
|
end
|
176
178
|
|
177
179
|
# Calculates an average poll interval based on the number of known Sidekiq processes.
|
178
180
|
# This minimizes a single point of failure by dispersing check-ins but without taxing
|
179
181
|
# Redis if you run many Sidekiq processes.
|
180
|
-
def scaled_poll_interval
|
181
|
-
process_count *
|
182
|
+
def scaled_poll_interval(process_count)
|
183
|
+
process_count * @config[:average_scheduled_poll_interval]
|
182
184
|
end
|
183
185
|
|
184
186
|
def process_count
|
185
|
-
|
186
|
-
# expensive at scale. Cut it down by 90% with this counter.
|
187
|
-
# NB: This method is only called by the scheduler thread so we
|
188
|
-
# don't need to worry about the thread safety of +=.
|
189
|
-
pcount = Sidekiq::ProcessSet.new(@count_calls % 10 == 0).size
|
187
|
+
pcount = Sidekiq.redis { |conn| conn.scard("processes") }
|
190
188
|
pcount = 1 if pcount == 0
|
191
|
-
@count_calls += 1
|
192
189
|
pcount
|
193
190
|
end
|
194
191
|
|
192
|
+
# A copy of Sidekiq::ProcessSet#cleanup because server
|
193
|
+
# should never depend on sidekiq/api.
|
194
|
+
def cleanup
|
195
|
+
# dont run cleanup more than once per minute
|
196
|
+
return 0 unless redis { |conn| conn.set("process_cleanup", "1", nx: true, ex: 60) }
|
197
|
+
|
198
|
+
count = 0
|
199
|
+
redis do |conn|
|
200
|
+
procs = conn.sscan("processes").to_a
|
201
|
+
heartbeats = conn.pipelined { |pipeline|
|
202
|
+
procs.each do |key|
|
203
|
+
pipeline.hget(key, "info")
|
204
|
+
end
|
205
|
+
}
|
206
|
+
|
207
|
+
# the hash named key has an expiry of 60 seconds.
|
208
|
+
# if it's not found, that means the process has not reported
|
209
|
+
# in to Redis and probably died.
|
210
|
+
to_prune = procs.select.with_index { |proc, i|
|
211
|
+
heartbeats[i].nil?
|
212
|
+
}
|
213
|
+
count = conn.srem("processes", to_prune) unless to_prune.empty?
|
214
|
+
end
|
215
|
+
count
|
216
|
+
end
|
217
|
+
|
195
218
|
def initial_wait
|
196
|
-
# Have all processes sleep between 5-15 seconds.
|
197
|
-
#
|
219
|
+
# Have all processes sleep between 5-15 seconds. 10 seconds to give time for
|
220
|
+
# the heartbeat to register (if the poll interval is going to be calculated by the number
|
198
221
|
# of workers), and 5 random seconds to ensure they don't all hit Redis at the same time.
|
199
222
|
total = 0
|
200
|
-
total += INITIAL_WAIT unless
|
223
|
+
total += INITIAL_WAIT unless @config[:poll_interval_average]
|
201
224
|
total += (5 * rand)
|
202
225
|
|
203
226
|
@sleeper.pop(total)
|
204
227
|
rescue Timeout::Error
|
228
|
+
ensure
|
229
|
+
# periodically clean out the `processes` set in Redis which can collect
|
230
|
+
# references to dead processes over time. The process count affects how
|
231
|
+
# often we scan for scheduled jobs.
|
232
|
+
cleanup
|
205
233
|
end
|
206
234
|
end
|
207
235
|
end
|
@@ -4,7 +4,7 @@ require "sidekiq/testing"
|
|
4
4
|
|
5
5
|
##
|
6
6
|
# The Sidekiq inline infrastructure overrides perform_async so that it
|
7
|
-
# actually calls perform instead. This allows
|
7
|
+
# actually calls perform instead. This allows jobs to be run inline in a
|
8
8
|
# testing environment.
|
9
9
|
#
|
10
10
|
# This is similar to `Resque.inline = true` functionality.
|
@@ -15,8 +15,8 @@ require "sidekiq/testing"
|
|
15
15
|
#
|
16
16
|
# $external_variable = 0
|
17
17
|
#
|
18
|
-
# class
|
19
|
-
# include Sidekiq::
|
18
|
+
# class ExternalJob
|
19
|
+
# include Sidekiq::Job
|
20
20
|
#
|
21
21
|
# def perform
|
22
22
|
# $external_variable = 1
|
@@ -24,7 +24,7 @@ require "sidekiq/testing"
|
|
24
24
|
# end
|
25
25
|
#
|
26
26
|
# assert_equal 0, $external_variable
|
27
|
-
#
|
27
|
+
# ExternalJob.perform_async
|
28
28
|
# assert_equal 1, $external_variable
|
29
29
|
#
|
30
30
|
Sidekiq::Testing.inline!
|
data/lib/sidekiq/testing.rb
CHANGED
@@ -51,19 +51,10 @@ module Sidekiq
|
|
51
51
|
end
|
52
52
|
|
53
53
|
def server_middleware
|
54
|
-
@server_chain ||= Middleware::Chain.new
|
54
|
+
@server_chain ||= Middleware::Chain.new(Sidekiq.default_configuration)
|
55
55
|
yield @server_chain if block_given?
|
56
56
|
@server_chain
|
57
57
|
end
|
58
|
-
|
59
|
-
def constantize(str)
|
60
|
-
names = str.split("::")
|
61
|
-
names.shift if names.empty? || names.first.empty?
|
62
|
-
|
63
|
-
names.inject(Object) do |constant, name|
|
64
|
-
constant.const_defined?(name) ? constant.const_get(name) : constant.const_missing(name)
|
65
|
-
end
|
66
|
-
end
|
67
58
|
end
|
68
59
|
end
|
69
60
|
|
@@ -83,7 +74,7 @@ module Sidekiq
|
|
83
74
|
true
|
84
75
|
elsif Sidekiq::Testing.inline?
|
85
76
|
payloads.each do |job|
|
86
|
-
klass =
|
77
|
+
klass = Object.const_get(job["class"])
|
87
78
|
job["id"] ||= SecureRandom.hex(12)
|
88
79
|
job_hash = Sidekiq.load_json(Sidekiq.dump_json(job))
|
89
80
|
klass.process_job(job_hash)
|
@@ -101,20 +92,20 @@ module Sidekiq
|
|
101
92
|
##
|
102
93
|
# The Queues class is only for testing the fake queue implementation.
|
103
94
|
# There are 2 data structures involved in tandem. This is due to the
|
104
|
-
# Rspec syntax of change(
|
95
|
+
# Rspec syntax of change(HardJob.jobs, :size). It keeps a reference
|
105
96
|
# to the array. Because the array was dervied from a filter of the total
|
106
97
|
# jobs enqueued, it appeared as though the array didn't change.
|
107
98
|
#
|
108
99
|
# To solve this, we'll keep 2 hashes containing the jobs. One with keys based
|
109
|
-
# on the queue, and another with keys of the
|
110
|
-
#
|
100
|
+
# on the queue, and another with keys of the job type, so the array for
|
101
|
+
# HardJob.jobs is a straight reference to a real array.
|
111
102
|
#
|
112
103
|
# Queue-based hash:
|
113
104
|
#
|
114
105
|
# {
|
115
106
|
# "default"=>[
|
116
107
|
# {
|
117
|
-
# "class"=>"TestTesting::
|
108
|
+
# "class"=>"TestTesting::HardJob",
|
118
109
|
# "args"=>[1, 2],
|
119
110
|
# "retry"=>true,
|
120
111
|
# "queue"=>"default",
|
@@ -124,12 +115,12 @@ module Sidekiq
|
|
124
115
|
# ]
|
125
116
|
# }
|
126
117
|
#
|
127
|
-
#
|
118
|
+
# Job-based hash:
|
128
119
|
#
|
129
120
|
# {
|
130
|
-
# "TestTesting::
|
121
|
+
# "TestTesting::HardJob"=>[
|
131
122
|
# {
|
132
|
-
# "class"=>"TestTesting::
|
123
|
+
# "class"=>"TestTesting::HardJob",
|
133
124
|
# "args"=>[1, 2],
|
134
125
|
# "retry"=>true,
|
135
126
|
# "queue"=>"default",
|
@@ -144,14 +135,14 @@ module Sidekiq
|
|
144
135
|
# require 'sidekiq/testing'
|
145
136
|
#
|
146
137
|
# assert_equal 0, Sidekiq::Queues["default"].size
|
147
|
-
#
|
138
|
+
# HardJob.perform_async(:something)
|
148
139
|
# assert_equal 1, Sidekiq::Queues["default"].size
|
149
140
|
# assert_equal :something, Sidekiq::Queues["default"].first['args'][0]
|
150
141
|
#
|
151
|
-
# You can also clear all
|
142
|
+
# You can also clear all jobs:
|
152
143
|
#
|
153
144
|
# assert_equal 0, Sidekiq::Queues["default"].size
|
154
|
-
#
|
145
|
+
# HardJob.perform_async(:something)
|
155
146
|
# Sidekiq::Queues.clear_all
|
156
147
|
# assert_equal 0, Sidekiq::Queues["default"].size
|
157
148
|
#
|
@@ -170,35 +161,36 @@ module Sidekiq
|
|
170
161
|
|
171
162
|
def push(queue, klass, job)
|
172
163
|
jobs_by_queue[queue] << job
|
173
|
-
|
164
|
+
jobs_by_class[klass] << job
|
174
165
|
end
|
175
166
|
|
176
167
|
def jobs_by_queue
|
177
168
|
@jobs_by_queue ||= Hash.new { |hash, key| hash[key] = [] }
|
178
169
|
end
|
179
170
|
|
180
|
-
def
|
181
|
-
@
|
171
|
+
def jobs_by_class
|
172
|
+
@jobs_by_class ||= Hash.new { |hash, key| hash[key] = [] }
|
182
173
|
end
|
174
|
+
alias_method :jobs_by_worker, :jobs_by_class
|
183
175
|
|
184
176
|
def delete_for(jid, queue, klass)
|
185
177
|
jobs_by_queue[queue.to_s].delete_if { |job| job["jid"] == jid }
|
186
|
-
|
178
|
+
jobs_by_class[klass].delete_if { |job| job["jid"] == jid }
|
187
179
|
end
|
188
180
|
|
189
181
|
def clear_for(queue, klass)
|
190
|
-
jobs_by_queue[queue].clear
|
191
|
-
|
182
|
+
jobs_by_queue[queue.to_s].clear
|
183
|
+
jobs_by_class[klass].clear
|
192
184
|
end
|
193
185
|
|
194
186
|
def clear_all
|
195
187
|
jobs_by_queue.clear
|
196
|
-
|
188
|
+
jobs_by_class.clear
|
197
189
|
end
|
198
190
|
end
|
199
191
|
end
|
200
192
|
|
201
|
-
module
|
193
|
+
module Job
|
202
194
|
##
|
203
195
|
# The Sidekiq testing infrastructure overrides perform_async
|
204
196
|
# so that it does not actually touch the network. Instead it
|
@@ -212,43 +204,27 @@ module Sidekiq
|
|
212
204
|
#
|
213
205
|
# require 'sidekiq/testing'
|
214
206
|
#
|
215
|
-
# assert_equal 0,
|
216
|
-
#
|
217
|
-
# assert_equal 1,
|
218
|
-
# assert_equal :something,
|
219
|
-
#
|
220
|
-
# assert_equal 0, Sidekiq::Extensions::DelayedMailer.jobs.size
|
221
|
-
# MyMailer.delay.send_welcome_email('foo@example.com')
|
222
|
-
# assert_equal 1, Sidekiq::Extensions::DelayedMailer.jobs.size
|
207
|
+
# assert_equal 0, HardJob.jobs.size
|
208
|
+
# HardJob.perform_async(:something)
|
209
|
+
# assert_equal 1, HardJob.jobs.size
|
210
|
+
# assert_equal :something, HardJob.jobs[0]['args'][0]
|
223
211
|
#
|
224
|
-
# You can also clear and drain all
|
212
|
+
# You can also clear and drain all job types:
|
225
213
|
#
|
226
|
-
#
|
227
|
-
# assert_equal 0, Sidekiq::Extensions::DelayedModel.jobs.size
|
228
|
-
#
|
229
|
-
# MyMailer.delay.send_welcome_email('foo@example.com')
|
230
|
-
# MyModel.delay.do_something_hard
|
231
|
-
#
|
232
|
-
# assert_equal 1, Sidekiq::Extensions::DelayedMailer.jobs.size
|
233
|
-
# assert_equal 1, Sidekiq::Extensions::DelayedModel.jobs.size
|
234
|
-
#
|
235
|
-
# Sidekiq::Worker.clear_all # or .drain_all
|
236
|
-
#
|
237
|
-
# assert_equal 0, Sidekiq::Extensions::DelayedMailer.jobs.size
|
238
|
-
# assert_equal 0, Sidekiq::Extensions::DelayedModel.jobs.size
|
214
|
+
# Sidekiq::Job.clear_all # or .drain_all
|
239
215
|
#
|
240
216
|
# This can be useful to make sure jobs don't linger between tests:
|
241
217
|
#
|
242
218
|
# RSpec.configure do |config|
|
243
219
|
# config.before(:each) do
|
244
|
-
# Sidekiq::
|
220
|
+
# Sidekiq::Job.clear_all
|
245
221
|
# end
|
246
222
|
# end
|
247
223
|
#
|
248
224
|
# or for acceptance testing, i.e. with cucumber:
|
249
225
|
#
|
250
226
|
# AfterStep do
|
251
|
-
# Sidekiq::
|
227
|
+
# Sidekiq::Job.drain_all
|
252
228
|
# end
|
253
229
|
#
|
254
230
|
# When I sign up as "foo@example.com"
|
@@ -262,7 +238,7 @@ module Sidekiq
|
|
262
238
|
|
263
239
|
# Jobs queued for this worker
|
264
240
|
def jobs
|
265
|
-
Queues.
|
241
|
+
Queues.jobs_by_class[to_s]
|
266
242
|
end
|
267
243
|
|
268
244
|
# Clear all jobs for this worker
|
@@ -288,11 +264,11 @@ module Sidekiq
|
|
288
264
|
end
|
289
265
|
|
290
266
|
def process_job(job)
|
291
|
-
|
292
|
-
|
293
|
-
|
294
|
-
Sidekiq::Testing.server_middleware.invoke(
|
295
|
-
execute_job(
|
267
|
+
inst = new
|
268
|
+
inst.jid = job["jid"]
|
269
|
+
inst.bid = job["bid"] if inst.respond_to?(:bid=)
|
270
|
+
Sidekiq::Testing.server_middleware.invoke(inst, job, job["queue"]) do
|
271
|
+
execute_job(inst, job["args"])
|
296
272
|
end
|
297
273
|
end
|
298
274
|
|
@@ -306,18 +282,18 @@ module Sidekiq
|
|
306
282
|
Queues.jobs_by_queue.values.flatten
|
307
283
|
end
|
308
284
|
|
309
|
-
# Clear all queued jobs
|
285
|
+
# Clear all queued jobs
|
310
286
|
def clear_all
|
311
287
|
Queues.clear_all
|
312
288
|
end
|
313
289
|
|
314
|
-
# Drain all queued jobs
|
290
|
+
# Drain (execute) all queued jobs
|
315
291
|
def drain_all
|
316
292
|
while jobs.any?
|
317
|
-
|
293
|
+
job_classes = jobs.map { |job| job["class"] }.uniq
|
318
294
|
|
319
|
-
|
320
|
-
|
295
|
+
job_classes.each do |job_class|
|
296
|
+
Object.const_get(job_class).drain
|
321
297
|
end
|
322
298
|
end
|
323
299
|
end
|
@@ -328,13 +304,10 @@ module Sidekiq
|
|
328
304
|
def jobs_for(klass)
|
329
305
|
jobs.select do |job|
|
330
306
|
marshalled = job["args"][0]
|
331
|
-
marshalled.index(klass.to_s) && YAML.
|
307
|
+
marshalled.index(klass.to_s) && YAML.safe_load(marshalled)[0] == klass
|
332
308
|
end
|
333
309
|
end
|
334
310
|
end
|
335
|
-
|
336
|
-
Sidekiq::Extensions::DelayedMailer.extend(TestingExtensions) if defined?(Sidekiq::Extensions::DelayedMailer)
|
337
|
-
Sidekiq::Extensions::DelayedModel.extend(TestingExtensions) if defined?(Sidekiq::Extensions::DelayedModel)
|
338
311
|
end
|
339
312
|
|
340
313
|
if defined?(::Rails) && Rails.respond_to?(:env) && !Rails.env.test? && !$TESTING
|
@@ -0,0 +1,44 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require "securerandom"
|
4
|
+
require "sidekiq/client"
|
5
|
+
|
6
|
+
module Sidekiq
|
7
|
+
class TransactionAwareClient
|
8
|
+
def initialize(pool: nil, config: nil)
|
9
|
+
@redis_client = Client.new(pool: pool, config: config)
|
10
|
+
end
|
11
|
+
|
12
|
+
def push(item)
|
13
|
+
# pre-allocate the JID so we can return it immediately and
|
14
|
+
# save it to the database as part of the transaction.
|
15
|
+
item["jid"] ||= SecureRandom.hex(12)
|
16
|
+
AfterCommitEverywhere.after_commit { @redis_client.push(item) }
|
17
|
+
item["jid"]
|
18
|
+
end
|
19
|
+
|
20
|
+
##
|
21
|
+
# We don't provide transactionality for push_bulk because we don't want
|
22
|
+
# to hold potentially hundreds of thousands of job records in memory due to
|
23
|
+
# a long running enqueue process.
|
24
|
+
def push_bulk(items)
|
25
|
+
@redis_client.push_bulk(items)
|
26
|
+
end
|
27
|
+
end
|
28
|
+
end
|
29
|
+
|
30
|
+
##
|
31
|
+
# Use `Sidekiq.transactional_push!` in your sidekiq.rb initializer
|
32
|
+
module Sidekiq
|
33
|
+
def self.transactional_push!
|
34
|
+
begin
|
35
|
+
require "after_commit_everywhere"
|
36
|
+
rescue LoadError
|
37
|
+
raise %q(You need to add `gem "after_commit_everywhere"` to your Gemfile to use Sidekiq's transactional client)
|
38
|
+
end
|
39
|
+
|
40
|
+
Sidekiq.default_job_options["client_class"] = Sidekiq::TransactionAwareClient
|
41
|
+
Sidekiq::JobUtil::TRANSIENT_ATTRIBUTES << "client_class"
|
42
|
+
true
|
43
|
+
end
|
44
|
+
end
|
data/lib/sidekiq/version.rb
CHANGED
data/lib/sidekiq/web/action.rb
CHANGED
@@ -15,11 +15,11 @@ module Sidekiq
|
|
15
15
|
end
|
16
16
|
|
17
17
|
def halt(res)
|
18
|
-
throw :halt, [res, {"
|
18
|
+
throw :halt, [res, {"content-type" => "text/plain"}, [res.to_s]]
|
19
19
|
end
|
20
20
|
|
21
21
|
def redirect(location)
|
22
|
-
throw :halt, [302, {"
|
22
|
+
throw :halt, [302, {"location" => "#{request.base_url}#{location}"}, []]
|
23
23
|
end
|
24
24
|
|
25
25
|
def params
|
@@ -68,7 +68,7 @@ module Sidekiq
|
|
68
68
|
end
|
69
69
|
|
70
70
|
def json(payload)
|
71
|
-
[200, {"
|
71
|
+
[200, {"content-type" => "application/json", "cache-control" => "private, no-store"}, [Sidekiq.dump_json(payload)]]
|
72
72
|
end
|
73
73
|
|
74
74
|
def initialize(env, block)
|
@@ -20,6 +20,12 @@ module Sidekiq
|
|
20
20
|
"worker-src 'self'",
|
21
21
|
"base-uri 'self'"
|
22
22
|
].join("; ").freeze
|
23
|
+
METRICS_PERIODS = {
|
24
|
+
"1h" => 60,
|
25
|
+
"2h" => 120,
|
26
|
+
"4h" => 240,
|
27
|
+
"8h" => 480
|
28
|
+
}
|
23
29
|
|
24
30
|
def initialize(klass)
|
25
31
|
@klass = klass
|
@@ -60,17 +66,42 @@ module Sidekiq
|
|
60
66
|
erb(:dashboard)
|
61
67
|
end
|
62
68
|
|
69
|
+
get "/metrics" do
|
70
|
+
q = Sidekiq::Metrics::Query.new
|
71
|
+
@period = params[:period]
|
72
|
+
@periods = METRICS_PERIODS
|
73
|
+
minutes = @periods.fetch(@period, @periods.values.first)
|
74
|
+
@query_result = q.top_jobs(minutes: minutes)
|
75
|
+
erb(:metrics)
|
76
|
+
end
|
77
|
+
|
78
|
+
get "/metrics/:name" do
|
79
|
+
@name = route_params[:name]
|
80
|
+
@period = params[:period]
|
81
|
+
q = Sidekiq::Metrics::Query.new
|
82
|
+
@periods = METRICS_PERIODS
|
83
|
+
minutes = @periods.fetch(@period, @periods.values.first)
|
84
|
+
@query_result = q.for_job(@name, minutes: minutes)
|
85
|
+
erb(:metrics_for_job)
|
86
|
+
end
|
87
|
+
|
63
88
|
get "/busy" do
|
89
|
+
@count = (params["count"] || 100).to_i
|
90
|
+
(@current_page, @total_size, @workset) = page_items(workset, params["page"], @count)
|
91
|
+
|
64
92
|
erb(:busy)
|
65
93
|
end
|
66
94
|
|
67
95
|
post "/busy" do
|
68
96
|
if params["identity"]
|
69
|
-
|
70
|
-
|
71
|
-
|
97
|
+
pro = Sidekiq::ProcessSet[params["identity"]]
|
98
|
+
|
99
|
+
pro.quiet! if params["quiet"]
|
100
|
+
pro.stop! if params["stop"]
|
72
101
|
else
|
73
102
|
processes.each do |pro|
|
103
|
+
next if pro.embedded?
|
104
|
+
|
74
105
|
pro.quiet! if params["quiet"]
|
75
106
|
pro.stop! if params["stop"]
|
76
107
|
end
|
@@ -294,12 +325,12 @@ module Sidekiq
|
|
294
325
|
end
|
295
326
|
|
296
327
|
get "/stats/queues" do
|
297
|
-
json Sidekiq::Stats
|
328
|
+
json Sidekiq::Stats.new.queues
|
298
329
|
end
|
299
330
|
|
300
331
|
def call(env)
|
301
332
|
action = self.class.match(env)
|
302
|
-
return [404, {"
|
333
|
+
return [404, {"content-type" => "text/plain", "x-cascade" => "pass"}, ["Not Found"]] unless action
|
303
334
|
|
304
335
|
app = @klass
|
305
336
|
resp = catch(:halt) do
|
@@ -316,10 +347,10 @@ module Sidekiq
|
|
316
347
|
else
|
317
348
|
# rendered content goes here
|
318
349
|
headers = {
|
319
|
-
"
|
320
|
-
"
|
321
|
-
"
|
322
|
-
"
|
350
|
+
"content-type" => "text/html",
|
351
|
+
"cache-control" => "private, no-store",
|
352
|
+
"content-language" => action.locale,
|
353
|
+
"content-security-policy" => CSP_HEADER
|
323
354
|
}
|
324
355
|
# we'll let Rack calculate Content-Length for us.
|
325
356
|
[200, headers, [resp]]
|
@@ -143,7 +143,7 @@ module Sidekiq
|
|
143
143
|
one_time_pad = SecureRandom.random_bytes(token.length)
|
144
144
|
encrypted_token = xor_byte_strings(one_time_pad, token)
|
145
145
|
masked_token = one_time_pad + encrypted_token
|
146
|
-
Base64.
|
146
|
+
Base64.urlsafe_encode64(masked_token)
|
147
147
|
end
|
148
148
|
|
149
149
|
# Essentially the inverse of +mask_token+.
|
@@ -152,7 +152,7 @@ module Sidekiq
|
|
152
152
|
# value and decrypt it
|
153
153
|
token_length = masked_token.length / 2
|
154
154
|
one_time_pad = masked_token[0...token_length]
|
155
|
-
encrypted_token = masked_token[token_length
|
155
|
+
encrypted_token = masked_token[token_length..]
|
156
156
|
xor_byte_strings(one_time_pad, encrypted_token)
|
157
157
|
end
|
158
158
|
|
@@ -169,7 +169,7 @@ module Sidekiq
|
|
169
169
|
end
|
170
170
|
|
171
171
|
def decode_token(token)
|
172
|
-
Base64.
|
172
|
+
Base64.urlsafe_decode64(token)
|
173
173
|
end
|
174
174
|
|
175
175
|
def xor_byte_strings(s1, s2)
|