sidekiq 6.0.7 → 6.4.2
Sign up to get free protection for your applications and to get access to all the features.
Potentially problematic release.
This version of sidekiq might be problematic. Click here for more details.
- checksums.yaml +4 -4
- data/Changes.md +189 -2
- data/LICENSE +3 -3
- data/README.md +11 -10
- data/bin/sidekiq +8 -3
- data/bin/sidekiqload +57 -65
- data/bin/sidekiqmon +1 -1
- data/lib/generators/sidekiq/job_generator.rb +57 -0
- data/lib/generators/sidekiq/templates/{worker.rb.erb → job.rb.erb} +2 -2
- data/lib/generators/sidekiq/templates/{worker_spec.rb.erb → job_spec.rb.erb} +1 -1
- data/lib/generators/sidekiq/templates/{worker_test.rb.erb → job_test.rb.erb} +1 -1
- data/lib/sidekiq/api.rb +164 -116
- data/lib/sidekiq/cli.rb +49 -15
- data/lib/sidekiq/client.rb +51 -70
- data/lib/sidekiq/delay.rb +2 -0
- data/lib/sidekiq/extensions/action_mailer.rb +3 -2
- data/lib/sidekiq/extensions/active_record.rb +4 -3
- data/lib/sidekiq/extensions/class_methods.rb +5 -4
- data/lib/sidekiq/extensions/generic_proxy.rb +4 -2
- data/lib/sidekiq/fetch.rb +32 -23
- data/lib/sidekiq/job.rb +13 -0
- data/lib/sidekiq/job_logger.rb +16 -28
- data/lib/sidekiq/job_retry.rb +32 -33
- data/lib/sidekiq/job_util.rb +67 -0
- data/lib/sidekiq/launcher.rb +113 -54
- data/lib/sidekiq/logger.rb +11 -20
- data/lib/sidekiq/manager.rb +16 -18
- data/lib/sidekiq/middleware/chain.rb +10 -8
- data/lib/sidekiq/middleware/current_attributes.rb +57 -0
- data/lib/sidekiq/middleware/i18n.rb +4 -4
- data/lib/sidekiq/monitor.rb +1 -1
- data/lib/sidekiq/paginator.rb +8 -8
- data/lib/sidekiq/processor.rb +31 -31
- data/lib/sidekiq/rails.rb +36 -20
- data/lib/sidekiq/redis_connection.rb +16 -15
- data/lib/sidekiq/scheduled.rb +51 -16
- data/lib/sidekiq/sd_notify.rb +1 -1
- data/lib/sidekiq/testing/inline.rb +4 -4
- data/lib/sidekiq/testing.rb +38 -39
- data/lib/sidekiq/util.rb +41 -0
- data/lib/sidekiq/version.rb +1 -1
- data/lib/sidekiq/web/action.rb +2 -2
- data/lib/sidekiq/web/application.rb +21 -12
- data/lib/sidekiq/web/csrf_protection.rb +180 -0
- data/lib/sidekiq/web/helpers.rb +39 -33
- data/lib/sidekiq/web/router.rb +5 -2
- data/lib/sidekiq/web.rb +36 -72
- data/lib/sidekiq/worker.rb +135 -16
- data/lib/sidekiq.rb +33 -17
- data/sidekiq.gemspec +11 -4
- data/web/assets/images/apple-touch-icon.png +0 -0
- data/web/assets/javascripts/application.js +113 -65
- data/web/assets/javascripts/dashboard.js +51 -51
- data/web/assets/stylesheets/application-dark.css +64 -43
- data/web/assets/stylesheets/application-rtl.css +0 -4
- data/web/assets/stylesheets/application.css +42 -239
- data/web/locales/ar.yml +8 -2
- data/web/locales/en.yml +4 -1
- data/web/locales/es.yml +18 -2
- data/web/locales/fr.yml +8 -1
- data/web/locales/ja.yml +3 -0
- data/web/locales/lt.yml +1 -1
- data/web/locales/pl.yml +4 -4
- data/web/locales/ru.yml +4 -0
- data/web/views/_footer.erb +1 -1
- data/web/views/_job_info.erb +1 -1
- data/web/views/_poll_link.erb +2 -5
- data/web/views/_summary.erb +7 -7
- data/web/views/busy.erb +51 -20
- data/web/views/dashboard.erb +22 -14
- data/web/views/dead.erb +1 -1
- data/web/views/layout.erb +2 -1
- data/web/views/morgue.erb +6 -6
- data/web/views/queue.erb +11 -11
- data/web/views/queues.erb +4 -4
- data/web/views/retries.erb +7 -7
- data/web/views/retry.erb +1 -1
- data/web/views/scheduled.erb +1 -1
- metadata +24 -49
- data/.circleci/config.yml +0 -60
- data/.github/contributing.md +0 -32
- data/.github/issue_template.md +0 -11
- data/.gitignore +0 -13
- data/.standard.yml +0 -20
- data/3.0-Upgrade.md +0 -70
- data/4.0-Upgrade.md +0 -53
- data/5.0-Upgrade.md +0 -56
- data/6.0-Upgrade.md +0 -72
- data/COMM-LICENSE +0 -97
- data/Ent-2.0-Upgrade.md +0 -37
- data/Ent-Changes.md +0 -256
- data/Gemfile +0 -24
- data/Gemfile.lock +0 -208
- data/Pro-2.0-Upgrade.md +0 -138
- data/Pro-3.0-Upgrade.md +0 -44
- data/Pro-4.0-Upgrade.md +0 -35
- data/Pro-5.0-Upgrade.md +0 -25
- data/Pro-Changes.md +0 -782
- data/Rakefile +0 -10
- data/code_of_conduct.md +0 -50
- data/lib/generators/sidekiq/worker_generator.rb +0 -57
data/lib/sidekiq/processor.rb
CHANGED
@@ -11,7 +11,7 @@ module Sidekiq
|
|
11
11
|
#
|
12
12
|
# 1. fetches a job from Redis
|
13
13
|
# 2. executes the job
|
14
|
-
# a. instantiate the
|
14
|
+
# a. instantiate the job class
|
15
15
|
# b. run the middleware chain
|
16
16
|
# c. call #perform
|
17
17
|
#
|
@@ -28,15 +28,15 @@ module Sidekiq
|
|
28
28
|
attr_reader :thread
|
29
29
|
attr_reader :job
|
30
30
|
|
31
|
-
def initialize(mgr)
|
31
|
+
def initialize(mgr, options)
|
32
32
|
@mgr = mgr
|
33
33
|
@down = false
|
34
34
|
@done = false
|
35
35
|
@job = nil
|
36
36
|
@thread = nil
|
37
|
-
@strategy =
|
38
|
-
@reloader =
|
39
|
-
@job_logger = (
|
37
|
+
@strategy = options[:fetch]
|
38
|
+
@reloader = options[:reloader] || proc { |&block| block.call }
|
39
|
+
@job_logger = (options[:job_logger] || Sidekiq::JobLogger).new
|
40
40
|
@retrier = Sidekiq::JobRetry.new
|
41
41
|
end
|
42
42
|
|
@@ -80,12 +80,12 @@ module Sidekiq
|
|
80
80
|
end
|
81
81
|
|
82
82
|
def get_one
|
83
|
-
|
83
|
+
uow = @strategy.retrieve_work
|
84
84
|
if @down
|
85
85
|
logger.info { "Redis is online, #{::Process.clock_gettime(::Process::CLOCK_MONOTONIC) - @down} sec downtime" }
|
86
86
|
@down = nil
|
87
87
|
end
|
88
|
-
|
88
|
+
uow
|
89
89
|
rescue Sidekiq::Shutdown
|
90
90
|
rescue => ex
|
91
91
|
handle_fetch_exception(ex)
|
@@ -130,10 +130,10 @@ module Sidekiq
|
|
130
130
|
# Effectively this block denotes a "unit of work" to Rails.
|
131
131
|
@reloader.call do
|
132
132
|
klass = constantize(job_hash["class"])
|
133
|
-
|
134
|
-
|
135
|
-
@retrier.local(
|
136
|
-
yield
|
133
|
+
inst = klass.new
|
134
|
+
inst.jid = job_hash["jid"]
|
135
|
+
@retrier.local(inst, jobstr, queue) do
|
136
|
+
yield inst
|
137
137
|
end
|
138
138
|
end
|
139
139
|
end
|
@@ -142,9 +142,9 @@ module Sidekiq
|
|
142
142
|
end
|
143
143
|
end
|
144
144
|
|
145
|
-
def process(
|
146
|
-
jobstr =
|
147
|
-
queue =
|
145
|
+
def process(uow)
|
146
|
+
jobstr = uow.job
|
147
|
+
queue = uow.queue_name
|
148
148
|
|
149
149
|
# Treat malformed JSON as a special case: job goes straight to the morgue.
|
150
150
|
job_hash = nil
|
@@ -154,14 +154,14 @@ module Sidekiq
|
|
154
154
|
handle_exception(ex, {context: "Invalid JSON for job", jobstr: jobstr})
|
155
155
|
# we can't notify because the job isn't a valid hash payload.
|
156
156
|
DeadSet.new.kill(jobstr, notify_failure: false)
|
157
|
-
return
|
157
|
+
return uow.acknowledge
|
158
158
|
end
|
159
159
|
|
160
160
|
ack = false
|
161
161
|
begin
|
162
|
-
dispatch(job_hash, queue, jobstr) do |
|
163
|
-
Sidekiq.server_middleware.invoke(
|
164
|
-
execute_job(
|
162
|
+
dispatch(job_hash, queue, jobstr) do |inst|
|
163
|
+
Sidekiq.server_middleware.invoke(inst, job_hash, queue) do
|
164
|
+
execute_job(inst, job_hash["args"])
|
165
165
|
end
|
166
166
|
end
|
167
167
|
ack = true
|
@@ -186,14 +186,14 @@ module Sidekiq
|
|
186
186
|
if ack
|
187
187
|
# We don't want a shutdown signal to interrupt job acknowledgment.
|
188
188
|
Thread.handle_interrupt(Sidekiq::Shutdown => :never) do
|
189
|
-
|
189
|
+
uow.acknowledge
|
190
190
|
end
|
191
191
|
end
|
192
192
|
end
|
193
193
|
end
|
194
194
|
|
195
|
-
def execute_job(
|
196
|
-
|
195
|
+
def execute_job(inst, cloned_args)
|
196
|
+
inst.perform(*cloned_args)
|
197
197
|
end
|
198
198
|
|
199
199
|
# Ruby doesn't provide atomic counters out of the box so we'll
|
@@ -219,39 +219,39 @@ module Sidekiq
|
|
219
219
|
end
|
220
220
|
|
221
221
|
# jruby's Hash implementation is not threadsafe, so we wrap it in a mutex here
|
222
|
-
class
|
222
|
+
class SharedWorkState
|
223
223
|
def initialize
|
224
|
-
@
|
224
|
+
@work_state = {}
|
225
225
|
@lock = Mutex.new
|
226
226
|
end
|
227
227
|
|
228
228
|
def set(tid, hash)
|
229
|
-
@lock.synchronize { @
|
229
|
+
@lock.synchronize { @work_state[tid] = hash }
|
230
230
|
end
|
231
231
|
|
232
232
|
def delete(tid)
|
233
|
-
@lock.synchronize { @
|
233
|
+
@lock.synchronize { @work_state.delete(tid) }
|
234
234
|
end
|
235
235
|
|
236
236
|
def dup
|
237
|
-
@lock.synchronize { @
|
237
|
+
@lock.synchronize { @work_state.dup }
|
238
238
|
end
|
239
239
|
|
240
240
|
def size
|
241
|
-
@lock.synchronize { @
|
241
|
+
@lock.synchronize { @work_state.size }
|
242
242
|
end
|
243
243
|
|
244
244
|
def clear
|
245
|
-
@lock.synchronize { @
|
245
|
+
@lock.synchronize { @work_state.clear }
|
246
246
|
end
|
247
247
|
end
|
248
248
|
|
249
249
|
PROCESSED = Counter.new
|
250
250
|
FAILURE = Counter.new
|
251
|
-
|
251
|
+
WORK_STATE = SharedWorkState.new
|
252
252
|
|
253
253
|
def stats(jobstr, queue)
|
254
|
-
|
254
|
+
WORK_STATE.set(tid, {queue: queue, payload: jobstr, run_at: Time.now.to_i})
|
255
255
|
|
256
256
|
begin
|
257
257
|
yield
|
@@ -259,7 +259,7 @@ module Sidekiq
|
|
259
259
|
FAILURE.incr
|
260
260
|
raise
|
261
261
|
ensure
|
262
|
-
|
262
|
+
WORK_STATE.delete(tid)
|
263
263
|
PROCESSED.incr
|
264
264
|
end
|
265
265
|
end
|
data/lib/sidekiq/rails.rb
CHANGED
@@ -1,9 +1,25 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
-
require "sidekiq/
|
3
|
+
require "sidekiq/job"
|
4
4
|
|
5
5
|
module Sidekiq
|
6
6
|
class Rails < ::Rails::Engine
|
7
|
+
class Reloader
|
8
|
+
def initialize(app = ::Rails.application)
|
9
|
+
@app = app
|
10
|
+
end
|
11
|
+
|
12
|
+
def call
|
13
|
+
@app.reloader.wrap do
|
14
|
+
yield
|
15
|
+
end
|
16
|
+
end
|
17
|
+
|
18
|
+
def inspect
|
19
|
+
"#<Sidekiq::Rails::Reloader @app=#{@app.class.name}>"
|
20
|
+
end
|
21
|
+
end
|
22
|
+
|
7
23
|
# By including the Options module, we allow AJs to directly control sidekiq features
|
8
24
|
# via the *sidekiq_options* class method and, for instance, not use AJ's retry system.
|
9
25
|
# AJ retries don't show up in the Sidekiq UI Retries tab, save any error data, can't be
|
@@ -17,14 +33,30 @@ module Sidekiq
|
|
17
33
|
# end
|
18
34
|
initializer "sidekiq.active_job_integration" do
|
19
35
|
ActiveSupport.on_load(:active_job) do
|
20
|
-
include ::Sidekiq::
|
36
|
+
include ::Sidekiq::Job::Options unless respond_to?(:sidekiq_options)
|
21
37
|
end
|
22
38
|
end
|
23
39
|
|
40
|
+
initializer "sidekiq.rails_logger" do
|
41
|
+
Sidekiq.configure_server do |_|
|
42
|
+
# This is the integration code necessary so that if a job uses `Rails.logger.info "Hello"`,
|
43
|
+
# it will appear in the Sidekiq console with all of the job context. See #5021 and
|
44
|
+
# https://github.com/rails/rails/blob/b5f2b550f69a99336482739000c58e4e04e033aa/railties/lib/rails/commands/server/server_command.rb#L82-L84
|
45
|
+
unless ::Rails.logger == ::Sidekiq.logger || ::ActiveSupport::Logger.logger_outputs_to?(::Rails.logger, $stdout)
|
46
|
+
::Rails.logger.extend(::ActiveSupport::Logger.broadcast(::Sidekiq.logger))
|
47
|
+
end
|
48
|
+
end
|
49
|
+
end
|
50
|
+
|
51
|
+
config.before_configuration do
|
52
|
+
dep = ActiveSupport::Deprecation.new("7.0", "Sidekiq")
|
53
|
+
dep.deprecate_methods(Sidekiq.singleton_class,
|
54
|
+
default_worker_options: :default_job_options,
|
55
|
+
"default_worker_options=": :default_job_options=)
|
56
|
+
end
|
57
|
+
|
24
58
|
# This hook happens after all initializers are run, just before returning
|
25
59
|
# from config/environment.rb back to sidekiq/cli.rb.
|
26
|
-
# We have to add the reloader after initialize to see if cache_classes has
|
27
|
-
# been turned on.
|
28
60
|
#
|
29
61
|
# None of this matters on the client-side, only within the Sidekiq process itself.
|
30
62
|
config.after_initialize do
|
@@ -32,21 +64,5 @@ module Sidekiq
|
|
32
64
|
Sidekiq.options[:reloader] = Sidekiq::Rails::Reloader.new
|
33
65
|
end
|
34
66
|
end
|
35
|
-
|
36
|
-
class Reloader
|
37
|
-
def initialize(app = ::Rails.application)
|
38
|
-
@app = app
|
39
|
-
end
|
40
|
-
|
41
|
-
def call
|
42
|
-
@app.reloader.wrap do
|
43
|
-
yield
|
44
|
-
end
|
45
|
-
end
|
46
|
-
|
47
|
-
def inspect
|
48
|
-
"#<Sidekiq::Rails::Reloader @app=#{@app.class.name}>"
|
49
|
-
end
|
50
|
-
end
|
51
67
|
end
|
52
68
|
end
|
@@ -8,16 +8,14 @@ module Sidekiq
|
|
8
8
|
class RedisConnection
|
9
9
|
class << self
|
10
10
|
def create(options = {})
|
11
|
-
options.
|
12
|
-
options[key.to_sym] = options.delete(key)
|
13
|
-
end
|
11
|
+
symbolized_options = options.transform_keys(&:to_sym)
|
14
12
|
|
15
|
-
if !
|
16
|
-
|
13
|
+
if !symbolized_options[:url] && (u = determine_redis_provider)
|
14
|
+
symbolized_options[:url] = u
|
17
15
|
end
|
18
16
|
|
19
|
-
size = if
|
20
|
-
|
17
|
+
size = if symbolized_options[:size]
|
18
|
+
symbolized_options[:size]
|
21
19
|
elsif Sidekiq.server?
|
22
20
|
# Give ourselves plenty of connections. pool is lazy
|
23
21
|
# so we won't create them until we need them.
|
@@ -30,17 +28,17 @@ module Sidekiq
|
|
30
28
|
|
31
29
|
verify_sizing(size, Sidekiq.options[:concurrency]) if Sidekiq.server?
|
32
30
|
|
33
|
-
pool_timeout =
|
34
|
-
log_info(
|
31
|
+
pool_timeout = symbolized_options[:pool_timeout] || 1
|
32
|
+
log_info(symbolized_options)
|
35
33
|
|
36
34
|
ConnectionPool.new(timeout: pool_timeout, size: size) do
|
37
|
-
build_client(
|
35
|
+
build_client(symbolized_options)
|
38
36
|
end
|
39
37
|
end
|
40
38
|
|
41
39
|
private
|
42
40
|
|
43
|
-
# Sidekiq needs
|
41
|
+
# Sidekiq needs many concurrent Redis connections.
|
44
42
|
#
|
45
43
|
# We need a connection for each Processor.
|
46
44
|
# We need a connection for Pro's real-time change listener
|
@@ -49,7 +47,7 @@ module Sidekiq
|
|
49
47
|
# - enterprise's leader election
|
50
48
|
# - enterprise's cron support
|
51
49
|
def verify_sizing(size, concurrency)
|
52
|
-
raise ArgumentError, "Your Redis connection pool is too small for Sidekiq
|
50
|
+
raise ArgumentError, "Your Redis connection pool is too small for Sidekiq. Your pool has #{size} connections but must have at least #{concurrency + 2}" if size < (concurrency + 2)
|
53
51
|
end
|
54
52
|
|
55
53
|
def build_client(options)
|
@@ -96,8 +94,11 @@ module Sidekiq
|
|
96
94
|
def log_info(options)
|
97
95
|
redacted = "REDACTED"
|
98
96
|
|
99
|
-
#
|
100
|
-
|
97
|
+
# Deep clone so we can muck with these options all we want and exclude
|
98
|
+
# params from dump-and-load that may contain objects that Marshal is
|
99
|
+
# unable to safely dump.
|
100
|
+
keys = options.keys - [:logger, :ssl_params]
|
101
|
+
scrubbed_options = Marshal.load(Marshal.dump(options.slice(*keys)))
|
101
102
|
if scrubbed_options[:url] && (uri = URI.parse(scrubbed_options[:url])) && uri.password
|
102
103
|
uri.password = redacted
|
103
104
|
scrubbed_options[:url] = uri.to_s
|
@@ -124,7 +125,7 @@ module Sidekiq
|
|
124
125
|
# initialization code at all.
|
125
126
|
#
|
126
127
|
p = ENV["REDIS_PROVIDER"]
|
127
|
-
if p && p =~
|
128
|
+
if p && p =~ /:/
|
128
129
|
raise <<~EOM
|
129
130
|
REDIS_PROVIDER should be set to the name of the variable which contains the Redis URL, not a URL itself.
|
130
131
|
Platforms like Heroku will sell addons that publish a *_URL variable. You need to tell Sidekiq with REDIS_PROVIDER, e.g.:
|
data/lib/sidekiq/scheduled.rb
CHANGED
@@ -9,29 +9,56 @@ module Sidekiq
|
|
9
9
|
SETS = %w[retry schedule]
|
10
10
|
|
11
11
|
class Enq
|
12
|
-
|
12
|
+
LUA_ZPOPBYSCORE = <<~LUA
|
13
|
+
local key, now = KEYS[1], ARGV[1]
|
14
|
+
local jobs = redis.call("zrangebyscore", key, "-inf", now, "limit", 0, 1)
|
15
|
+
if jobs[1] then
|
16
|
+
redis.call("zrem", key, jobs[1])
|
17
|
+
return jobs[1]
|
18
|
+
end
|
19
|
+
LUA
|
20
|
+
|
21
|
+
def initialize
|
22
|
+
@done = false
|
23
|
+
@lua_zpopbyscore_sha = nil
|
24
|
+
end
|
25
|
+
|
26
|
+
def enqueue_jobs(sorted_sets = SETS)
|
13
27
|
# A job's "score" in Redis is the time at which it should be processed.
|
14
28
|
# Just check Redis for the set of jobs with a timestamp before now.
|
15
29
|
Sidekiq.redis do |conn|
|
16
30
|
sorted_sets.each do |sorted_set|
|
17
|
-
# Get next
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
# the queue, it's because another process already popped it so we can move on to the
|
25
|
-
# next one.
|
26
|
-
if conn.zrem(sorted_set, job)
|
27
|
-
Sidekiq::Client.push(Sidekiq.load_json(job))
|
28
|
-
Sidekiq.logger.debug { "enqueued #{sorted_set}: #{job}" }
|
29
|
-
end
|
30
|
-
end
|
31
|
+
# Get next item in the queue with score (time to execute) <= now.
|
32
|
+
# We need to go through the list one at a time to reduce the risk of something
|
33
|
+
# going wrong between the time jobs are popped from the scheduled queue and when
|
34
|
+
# they are pushed onto a work queue and losing the jobs.
|
35
|
+
while !@done && (job = zpopbyscore(conn, keys: [sorted_set], argv: [Time.now.to_f.to_s]))
|
36
|
+
Sidekiq::Client.push(Sidekiq.load_json(job))
|
37
|
+
Sidekiq.logger.debug { "enqueued #{sorted_set}: #{job}" }
|
31
38
|
end
|
32
39
|
end
|
33
40
|
end
|
34
41
|
end
|
42
|
+
|
43
|
+
def terminate
|
44
|
+
@done = true
|
45
|
+
end
|
46
|
+
|
47
|
+
private
|
48
|
+
|
49
|
+
def zpopbyscore(conn, keys: nil, argv: nil)
|
50
|
+
if @lua_zpopbyscore_sha.nil?
|
51
|
+
raw_conn = conn.respond_to?(:redis) ? conn.redis : conn
|
52
|
+
@lua_zpopbyscore_sha = raw_conn.script(:load, LUA_ZPOPBYSCORE)
|
53
|
+
end
|
54
|
+
|
55
|
+
conn.evalsha(@lua_zpopbyscore_sha, keys: keys, argv: argv)
|
56
|
+
rescue Redis::CommandError => e
|
57
|
+
raise unless e.message.start_with?("NOSCRIPT")
|
58
|
+
|
59
|
+
@lua_zpopbyscore_sha = nil
|
60
|
+
retry
|
61
|
+
end
|
35
62
|
end
|
36
63
|
|
37
64
|
##
|
@@ -49,11 +76,14 @@ module Sidekiq
|
|
49
76
|
@sleeper = ConnectionPool::TimedStack.new
|
50
77
|
@done = false
|
51
78
|
@thread = nil
|
79
|
+
@count_calls = 0
|
52
80
|
end
|
53
81
|
|
54
82
|
# Shut down this instance, will pause until the thread is dead.
|
55
83
|
def terminate
|
56
84
|
@done = true
|
85
|
+
@enq.terminate if @enq.respond_to?(:terminate)
|
86
|
+
|
57
87
|
if @thread
|
58
88
|
t = @thread
|
59
89
|
@thread = nil
|
@@ -152,8 +182,13 @@ module Sidekiq
|
|
152
182
|
end
|
153
183
|
|
154
184
|
def process_count
|
155
|
-
|
185
|
+
# The work buried within Sidekiq::ProcessSet#cleanup can be
|
186
|
+
# expensive at scale. Cut it down by 90% with this counter.
|
187
|
+
# NB: This method is only called by the scheduler thread so we
|
188
|
+
# don't need to worry about the thread safety of +=.
|
189
|
+
pcount = Sidekiq::ProcessSet.new(@count_calls % 10 == 0).size
|
156
190
|
pcount = 1 if pcount == 0
|
191
|
+
@count_calls += 1
|
157
192
|
pcount
|
158
193
|
end
|
159
194
|
|
data/lib/sidekiq/sd_notify.rb
CHANGED
@@ -85,7 +85,7 @@ module Sidekiq
|
|
85
85
|
notify(FDSTORE, unset_env)
|
86
86
|
end
|
87
87
|
|
88
|
-
# @
|
88
|
+
# @return [Boolean] true if the service manager expects watchdog keep-alive
|
89
89
|
# notification messages to be sent from this process.
|
90
90
|
#
|
91
91
|
# If the $WATCHDOG_USEC environment variable is set,
|
@@ -4,7 +4,7 @@ require "sidekiq/testing"
|
|
4
4
|
|
5
5
|
##
|
6
6
|
# The Sidekiq inline infrastructure overrides perform_async so that it
|
7
|
-
# actually calls perform instead. This allows
|
7
|
+
# actually calls perform instead. This allows jobs to be run inline in a
|
8
8
|
# testing environment.
|
9
9
|
#
|
10
10
|
# This is similar to `Resque.inline = true` functionality.
|
@@ -15,8 +15,8 @@ require "sidekiq/testing"
|
|
15
15
|
#
|
16
16
|
# $external_variable = 0
|
17
17
|
#
|
18
|
-
# class
|
19
|
-
# include Sidekiq::
|
18
|
+
# class ExternalJob
|
19
|
+
# include Sidekiq::Job
|
20
20
|
#
|
21
21
|
# def perform
|
22
22
|
# $external_variable = 1
|
@@ -24,7 +24,7 @@ require "sidekiq/testing"
|
|
24
24
|
# end
|
25
25
|
#
|
26
26
|
# assert_equal 0, $external_variable
|
27
|
-
#
|
27
|
+
# ExternalJob.perform_async
|
28
28
|
# assert_equal 1, $external_variable
|
29
29
|
#
|
30
30
|
Sidekiq::Testing.inline!
|
data/lib/sidekiq/testing.rb
CHANGED
@@ -101,20 +101,20 @@ module Sidekiq
|
|
101
101
|
##
|
102
102
|
# The Queues class is only for testing the fake queue implementation.
|
103
103
|
# There are 2 data structures involved in tandem. This is due to the
|
104
|
-
# Rspec syntax of change(
|
104
|
+
# Rspec syntax of change(HardJob.jobs, :size). It keeps a reference
|
105
105
|
# to the array. Because the array was dervied from a filter of the total
|
106
106
|
# jobs enqueued, it appeared as though the array didn't change.
|
107
107
|
#
|
108
108
|
# To solve this, we'll keep 2 hashes containing the jobs. One with keys based
|
109
|
-
# on the queue, and another with keys of the
|
110
|
-
#
|
109
|
+
# on the queue, and another with keys of the job type, so the array for
|
110
|
+
# HardJob.jobs is a straight reference to a real array.
|
111
111
|
#
|
112
112
|
# Queue-based hash:
|
113
113
|
#
|
114
114
|
# {
|
115
115
|
# "default"=>[
|
116
116
|
# {
|
117
|
-
# "class"=>"TestTesting::
|
117
|
+
# "class"=>"TestTesting::HardJob",
|
118
118
|
# "args"=>[1, 2],
|
119
119
|
# "retry"=>true,
|
120
120
|
# "queue"=>"default",
|
@@ -124,12 +124,12 @@ module Sidekiq
|
|
124
124
|
# ]
|
125
125
|
# }
|
126
126
|
#
|
127
|
-
#
|
127
|
+
# Job-based hash:
|
128
128
|
#
|
129
129
|
# {
|
130
|
-
# "TestTesting::
|
130
|
+
# "TestTesting::HardJob"=>[
|
131
131
|
# {
|
132
|
-
# "class"=>"TestTesting::
|
132
|
+
# "class"=>"TestTesting::HardJob",
|
133
133
|
# "args"=>[1, 2],
|
134
134
|
# "retry"=>true,
|
135
135
|
# "queue"=>"default",
|
@@ -144,14 +144,14 @@ module Sidekiq
|
|
144
144
|
# require 'sidekiq/testing'
|
145
145
|
#
|
146
146
|
# assert_equal 0, Sidekiq::Queues["default"].size
|
147
|
-
#
|
147
|
+
# HardJob.perform_async(:something)
|
148
148
|
# assert_equal 1, Sidekiq::Queues["default"].size
|
149
149
|
# assert_equal :something, Sidekiq::Queues["default"].first['args'][0]
|
150
150
|
#
|
151
|
-
# You can also clear all
|
151
|
+
# You can also clear all jobs:
|
152
152
|
#
|
153
153
|
# assert_equal 0, Sidekiq::Queues["default"].size
|
154
|
-
#
|
154
|
+
# HardJob.perform_async(:something)
|
155
155
|
# Sidekiq::Queues.clear_all
|
156
156
|
# assert_equal 0, Sidekiq::Queues["default"].size
|
157
157
|
#
|
@@ -170,35 +170,36 @@ module Sidekiq
|
|
170
170
|
|
171
171
|
def push(queue, klass, job)
|
172
172
|
jobs_by_queue[queue] << job
|
173
|
-
|
173
|
+
jobs_by_class[klass] << job
|
174
174
|
end
|
175
175
|
|
176
176
|
def jobs_by_queue
|
177
177
|
@jobs_by_queue ||= Hash.new { |hash, key| hash[key] = [] }
|
178
178
|
end
|
179
179
|
|
180
|
-
def
|
181
|
-
@
|
180
|
+
def jobs_by_class
|
181
|
+
@jobs_by_class ||= Hash.new { |hash, key| hash[key] = [] }
|
182
182
|
end
|
183
|
+
alias_method :jobs_by_worker, :jobs_by_class
|
183
184
|
|
184
185
|
def delete_for(jid, queue, klass)
|
185
186
|
jobs_by_queue[queue.to_s].delete_if { |job| job["jid"] == jid }
|
186
|
-
|
187
|
+
jobs_by_class[klass].delete_if { |job| job["jid"] == jid }
|
187
188
|
end
|
188
189
|
|
189
190
|
def clear_for(queue, klass)
|
190
191
|
jobs_by_queue[queue].clear
|
191
|
-
|
192
|
+
jobs_by_class[klass].clear
|
192
193
|
end
|
193
194
|
|
194
195
|
def clear_all
|
195
196
|
jobs_by_queue.clear
|
196
|
-
|
197
|
+
jobs_by_class.clear
|
197
198
|
end
|
198
199
|
end
|
199
200
|
end
|
200
201
|
|
201
|
-
module
|
202
|
+
module Job
|
202
203
|
##
|
203
204
|
# The Sidekiq testing infrastructure overrides perform_async
|
204
205
|
# so that it does not actually touch the network. Instead it
|
@@ -212,16 +213,16 @@ module Sidekiq
|
|
212
213
|
#
|
213
214
|
# require 'sidekiq/testing'
|
214
215
|
#
|
215
|
-
# assert_equal 0,
|
216
|
-
#
|
217
|
-
# assert_equal 1,
|
218
|
-
# assert_equal :something,
|
216
|
+
# assert_equal 0, HardJob.jobs.size
|
217
|
+
# HardJob.perform_async(:something)
|
218
|
+
# assert_equal 1, HardJob.jobs.size
|
219
|
+
# assert_equal :something, HardJob.jobs[0]['args'][0]
|
219
220
|
#
|
220
221
|
# assert_equal 0, Sidekiq::Extensions::DelayedMailer.jobs.size
|
221
222
|
# MyMailer.delay.send_welcome_email('foo@example.com')
|
222
223
|
# assert_equal 1, Sidekiq::Extensions::DelayedMailer.jobs.size
|
223
224
|
#
|
224
|
-
# You can also clear and drain all
|
225
|
+
# You can also clear and drain all job types:
|
225
226
|
#
|
226
227
|
# assert_equal 0, Sidekiq::Extensions::DelayedMailer.jobs.size
|
227
228
|
# assert_equal 0, Sidekiq::Extensions::DelayedModel.jobs.size
|
@@ -241,14 +242,14 @@ module Sidekiq
|
|
241
242
|
#
|
242
243
|
# RSpec.configure do |config|
|
243
244
|
# config.before(:each) do
|
244
|
-
# Sidekiq::
|
245
|
+
# Sidekiq::Job.clear_all
|
245
246
|
# end
|
246
247
|
# end
|
247
248
|
#
|
248
249
|
# or for acceptance testing, i.e. with cucumber:
|
249
250
|
#
|
250
251
|
# AfterStep do
|
251
|
-
# Sidekiq::
|
252
|
+
# Sidekiq::Job.drain_all
|
252
253
|
# end
|
253
254
|
#
|
254
255
|
# When I sign up as "foo@example.com"
|
@@ -262,7 +263,7 @@ module Sidekiq
|
|
262
263
|
|
263
264
|
# Jobs queued for this worker
|
264
265
|
def jobs
|
265
|
-
Queues.
|
266
|
+
Queues.jobs_by_class[to_s]
|
266
267
|
end
|
267
268
|
|
268
269
|
# Clear all jobs for this worker
|
@@ -288,11 +289,11 @@ module Sidekiq
|
|
288
289
|
end
|
289
290
|
|
290
291
|
def process_job(job)
|
291
|
-
|
292
|
-
|
293
|
-
|
294
|
-
Sidekiq::Testing.server_middleware.invoke(
|
295
|
-
execute_job(
|
292
|
+
inst = new
|
293
|
+
inst.jid = job["jid"]
|
294
|
+
inst.bid = job["bid"] if inst.respond_to?(:bid=)
|
295
|
+
Sidekiq::Testing.server_middleware.invoke(inst, job, job["queue"]) do
|
296
|
+
execute_job(inst, job["args"])
|
296
297
|
end
|
297
298
|
end
|
298
299
|
|
@@ -306,18 +307,18 @@ module Sidekiq
|
|
306
307
|
Queues.jobs_by_queue.values.flatten
|
307
308
|
end
|
308
309
|
|
309
|
-
# Clear all queued jobs
|
310
|
+
# Clear all queued jobs
|
310
311
|
def clear_all
|
311
312
|
Queues.clear_all
|
312
313
|
end
|
313
314
|
|
314
|
-
# Drain all queued jobs
|
315
|
+
# Drain (execute) all queued jobs
|
315
316
|
def drain_all
|
316
317
|
while jobs.any?
|
317
|
-
|
318
|
+
job_classes = jobs.map { |job| job["class"] }.uniq
|
318
319
|
|
319
|
-
|
320
|
-
Sidekiq::Testing.constantize(
|
320
|
+
job_classes.each do |job_class|
|
321
|
+
Sidekiq::Testing.constantize(job_class).drain
|
321
322
|
end
|
322
323
|
end
|
323
324
|
end
|
@@ -337,8 +338,6 @@ module Sidekiq
|
|
337
338
|
Sidekiq::Extensions::DelayedModel.extend(TestingExtensions) if defined?(Sidekiq::Extensions::DelayedModel)
|
338
339
|
end
|
339
340
|
|
340
|
-
if defined?(::Rails) && Rails.respond_to?(:env) && !Rails.env.test?
|
341
|
-
|
342
|
-
puts("⛔️ WARNING: Sidekiq testing API enabled, but this is not the test environment. Your jobs will not go to Redis.")
|
343
|
-
puts("**************************************************")
|
341
|
+
if defined?(::Rails) && Rails.respond_to?(:env) && !Rails.env.test? && !$TESTING
|
342
|
+
warn("⛔️ WARNING: Sidekiq testing API enabled, but this is not the test environment. Your jobs will not go to Redis.", uplevel: 1)
|
344
343
|
end
|