sidekiq 6.4.1 → 6.5.0
Sign up to get free protection for your applications and to get access to all the features.
Potentially problematic release.
This version of sidekiq might be problematic. Click here for more details.
- checksums.yaml +4 -4
- data/Changes.md +43 -1
- data/README.md +1 -1
- data/bin/sidekiqload +16 -10
- data/lib/sidekiq/api.rb +42 -33
- data/lib/sidekiq/cli.rb +37 -36
- data/lib/sidekiq/client.rb +25 -26
- data/lib/sidekiq/component.rb +64 -0
- data/lib/sidekiq/extensions/generic_proxy.rb +1 -1
- data/lib/sidekiq/fetch.rb +15 -13
- data/lib/sidekiq/job_retry.rb +27 -26
- data/lib/sidekiq/job_util.rb +15 -9
- data/lib/sidekiq/launcher.rb +31 -29
- data/lib/sidekiq/logger.rb +5 -19
- data/lib/sidekiq/manager.rb +28 -25
- data/lib/sidekiq/middleware/chain.rb +22 -13
- data/lib/sidekiq/middleware/current_attributes.rb +4 -0
- data/lib/sidekiq/middleware/i18n.rb +6 -4
- data/lib/sidekiq/middleware/modules.rb +19 -0
- data/lib/sidekiq/monitor.rb +1 -1
- data/lib/sidekiq/paginator.rb +2 -2
- data/lib/sidekiq/processor.rb +38 -38
- data/lib/sidekiq/rails.rb +15 -8
- data/lib/sidekiq/redis_client_adapter.rb +154 -0
- data/lib/sidekiq/redis_connection.rb +81 -48
- data/lib/sidekiq/ring_buffer.rb +29 -0
- data/lib/sidekiq/scheduled.rb +11 -10
- data/lib/sidekiq/testing/inline.rb +4 -4
- data/lib/sidekiq/testing.rb +37 -36
- data/lib/sidekiq/transaction_aware_client.rb +45 -0
- data/lib/sidekiq/version.rb +1 -1
- data/lib/sidekiq/web/csrf_protection.rb +2 -2
- data/lib/sidekiq/web/helpers.rb +4 -4
- data/lib/sidekiq/worker.rb +18 -13
- data/lib/sidekiq.rb +97 -30
- data/web/assets/javascripts/application.js +58 -26
- data/web/assets/stylesheets/application.css +1 -2
- data/web/locales/pt-br.yml +27 -9
- data/web/views/_summary.erb +1 -1
- data/web/views/busy.erb +3 -3
- metadata +7 -4
- data/lib/sidekiq/exception_handler.rb +0 -27
- data/lib/sidekiq/util.rb +0 -108
@@ -0,0 +1,154 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require "connection_pool"
|
4
|
+
require "redis_client"
|
5
|
+
require "redis_client/decorator"
|
6
|
+
require "uri"
|
7
|
+
|
8
|
+
module Sidekiq
|
9
|
+
class RedisClientAdapter
|
10
|
+
BaseError = RedisClient::Error
|
11
|
+
CommandError = RedisClient::CommandError
|
12
|
+
|
13
|
+
module CompatMethods
|
14
|
+
def info
|
15
|
+
@client.call("INFO") { |i| i.lines(chomp: true).map { |l| l.split(":", 2) }.select { |l| l.size == 2 }.to_h }
|
16
|
+
end
|
17
|
+
|
18
|
+
def evalsha(sha, keys, argv)
|
19
|
+
@client.call("EVALSHA", sha, keys.size, *keys, *argv)
|
20
|
+
end
|
21
|
+
|
22
|
+
def brpoplpush(*args)
|
23
|
+
@client.blocking_call(false, "BRPOPLPUSH", *args)
|
24
|
+
end
|
25
|
+
|
26
|
+
def brpop(*args)
|
27
|
+
@client.blocking_call(false, "BRPOP", *args)
|
28
|
+
end
|
29
|
+
|
30
|
+
def set(*args)
|
31
|
+
@client.call("SET", *args) { |r| r == "OK" }
|
32
|
+
end
|
33
|
+
ruby2_keywords :set if respond_to?(:ruby2_keywords, true)
|
34
|
+
|
35
|
+
def sismember(*args)
|
36
|
+
@client.call("SISMEMBER", *args) { |c| c > 0 }
|
37
|
+
end
|
38
|
+
|
39
|
+
def exists?(key)
|
40
|
+
@client.call("EXISTS", key) { |c| c > 0 }
|
41
|
+
end
|
42
|
+
|
43
|
+
private
|
44
|
+
|
45
|
+
def method_missing(*args, &block)
|
46
|
+
@client.call(*args, *block)
|
47
|
+
end
|
48
|
+
ruby2_keywords :method_missing if respond_to?(:ruby2_keywords, true)
|
49
|
+
|
50
|
+
def respond_to_missing?(name, include_private = false)
|
51
|
+
super # Appease the linter. We can't tell what is a valid command.
|
52
|
+
end
|
53
|
+
end
|
54
|
+
|
55
|
+
CompatClient = RedisClient::Decorator.create(CompatMethods)
|
56
|
+
|
57
|
+
class CompatClient
|
58
|
+
%i[scan sscan zscan hscan].each do |method|
|
59
|
+
alias_method :"#{method}_each", method
|
60
|
+
undef_method method
|
61
|
+
end
|
62
|
+
|
63
|
+
def disconnect!
|
64
|
+
@client.close
|
65
|
+
end
|
66
|
+
|
67
|
+
def connection
|
68
|
+
{id: @client.id}
|
69
|
+
end
|
70
|
+
|
71
|
+
def redis
|
72
|
+
self
|
73
|
+
end
|
74
|
+
|
75
|
+
def _client
|
76
|
+
@client
|
77
|
+
end
|
78
|
+
|
79
|
+
def message
|
80
|
+
yield nil, @queue.pop
|
81
|
+
end
|
82
|
+
|
83
|
+
# NB: this method does not return
|
84
|
+
def subscribe(chan)
|
85
|
+
@queue = ::Queue.new
|
86
|
+
|
87
|
+
pubsub = @client.pubsub
|
88
|
+
pubsub.call("subscribe", chan)
|
89
|
+
|
90
|
+
loop do
|
91
|
+
evt = pubsub.next_event
|
92
|
+
next if evt.nil?
|
93
|
+
next unless evt[0] == "message" && evt[1] == chan
|
94
|
+
|
95
|
+
(_, _, msg) = evt
|
96
|
+
@queue << msg
|
97
|
+
yield self
|
98
|
+
end
|
99
|
+
end
|
100
|
+
end
|
101
|
+
|
102
|
+
def initialize(options)
|
103
|
+
opts = client_opts(options)
|
104
|
+
@config = if opts.key?(:sentinels)
|
105
|
+
RedisClient.sentinel(**opts)
|
106
|
+
else
|
107
|
+
RedisClient.config(**opts)
|
108
|
+
end
|
109
|
+
end
|
110
|
+
|
111
|
+
def new_client
|
112
|
+
CompatClient.new(@config.new_client)
|
113
|
+
end
|
114
|
+
|
115
|
+
private
|
116
|
+
|
117
|
+
def client_opts(options)
|
118
|
+
opts = options.dup
|
119
|
+
|
120
|
+
if opts[:namespace]
|
121
|
+
Sidekiq.logger.error("Your Redis configuration uses the namespace '#{opts[:namespace]}' but this feature isn't supported by redis-client. " \
|
122
|
+
"Either use the redis adapter or remove the namespace.")
|
123
|
+
Kernel.exit(-127)
|
124
|
+
end
|
125
|
+
|
126
|
+
opts.delete(:size)
|
127
|
+
opts.delete(:pool_timeout)
|
128
|
+
|
129
|
+
if opts[:network_timeout]
|
130
|
+
opts[:timeout] = opts[:network_timeout]
|
131
|
+
opts.delete(:network_timeout)
|
132
|
+
end
|
133
|
+
|
134
|
+
if opts[:driver]
|
135
|
+
opts[:driver] = opts[:driver].to_sym
|
136
|
+
end
|
137
|
+
|
138
|
+
opts[:name] = opts.delete(:master_name) if opts.key?(:master_name)
|
139
|
+
opts[:role] = opts[:role].to_sym if opts.key?(:role)
|
140
|
+
opts.delete(:url) if opts.key?(:sentinels)
|
141
|
+
|
142
|
+
# Issue #3303, redis-rb will silently retry an operation.
|
143
|
+
# This can lead to duplicate jobs if Sidekiq::Client's LPUSH
|
144
|
+
# is performed twice but I believe this is much, much rarer
|
145
|
+
# than the reconnect silently fixing a problem; we keep it
|
146
|
+
# on by default.
|
147
|
+
opts[:reconnect_attempts] ||= 1
|
148
|
+
|
149
|
+
opts
|
150
|
+
end
|
151
|
+
end
|
152
|
+
end
|
153
|
+
|
154
|
+
Sidekiq::RedisConnection.adapter = Sidekiq::RedisClientAdapter
|
@@ -5,55 +5,20 @@ require "redis"
|
|
5
5
|
require "uri"
|
6
6
|
|
7
7
|
module Sidekiq
|
8
|
-
|
9
|
-
class
|
10
|
-
|
11
|
-
|
12
|
-
|
13
|
-
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
-
size = if symbolized_options[:size]
|
18
|
-
symbolized_options[:size]
|
19
|
-
elsif Sidekiq.server?
|
20
|
-
# Give ourselves plenty of connections. pool is lazy
|
21
|
-
# so we won't create them until we need them.
|
22
|
-
Sidekiq.options[:concurrency] + 5
|
23
|
-
elsif ENV["RAILS_MAX_THREADS"]
|
24
|
-
Integer(ENV["RAILS_MAX_THREADS"])
|
25
|
-
else
|
26
|
-
5
|
27
|
-
end
|
28
|
-
|
29
|
-
verify_sizing(size, Sidekiq.options[:concurrency]) if Sidekiq.server?
|
30
|
-
|
31
|
-
pool_timeout = symbolized_options[:pool_timeout] || 1
|
32
|
-
log_info(symbolized_options)
|
33
|
-
|
34
|
-
ConnectionPool.new(timeout: pool_timeout, size: size) do
|
35
|
-
build_client(symbolized_options)
|
36
|
-
end
|
8
|
+
module RedisConnection
|
9
|
+
class RedisAdapter
|
10
|
+
BaseError = Redis::BaseError
|
11
|
+
CommandError = Redis::CommandError
|
12
|
+
|
13
|
+
def initialize(options)
|
14
|
+
warn("Usage of the 'redis' gem within Sidekiq itself is deprecated, Sidekiq 7.0 will only use the new, simpler 'redis-client' gem", caller) if ENV["SIDEKIQ_REDIS_CLIENT"] == "1"
|
15
|
+
@options = options
|
37
16
|
end
|
38
17
|
|
39
|
-
|
18
|
+
def new_client
|
19
|
+
namespace = @options[:namespace]
|
40
20
|
|
41
|
-
|
42
|
-
#
|
43
|
-
# We need a connection for each Processor.
|
44
|
-
# We need a connection for Pro's real-time change listener
|
45
|
-
# We need a connection to various features to call Redis every few seconds:
|
46
|
-
# - the process heartbeat.
|
47
|
-
# - enterprise's leader election
|
48
|
-
# - enterprise's cron support
|
49
|
-
def verify_sizing(size, concurrency)
|
50
|
-
raise ArgumentError, "Your Redis connection pool is too small for Sidekiq to work. Your pool has #{size} connections but must have at least #{concurrency + 2}" if size < (concurrency + 2)
|
51
|
-
end
|
52
|
-
|
53
|
-
def build_client(options)
|
54
|
-
namespace = options[:namespace]
|
55
|
-
|
56
|
-
client = Redis.new client_opts(options)
|
21
|
+
client = Redis.new client_opts(@options)
|
57
22
|
if namespace
|
58
23
|
begin
|
59
24
|
require "redis/namespace"
|
@@ -68,6 +33,8 @@ module Sidekiq
|
|
68
33
|
end
|
69
34
|
end
|
70
35
|
|
36
|
+
private
|
37
|
+
|
71
38
|
def client_opts(options)
|
72
39
|
opts = options.dup
|
73
40
|
if opts[:namespace]
|
@@ -90,6 +57,72 @@ module Sidekiq
|
|
90
57
|
|
91
58
|
opts
|
92
59
|
end
|
60
|
+
end
|
61
|
+
|
62
|
+
@adapter = RedisAdapter
|
63
|
+
|
64
|
+
class << self
|
65
|
+
attr_reader :adapter
|
66
|
+
|
67
|
+
# RedisConnection.adapter = :redis
|
68
|
+
# RedisConnection.adapter = :redis_client
|
69
|
+
def adapter=(adapter)
|
70
|
+
raise "no" if adapter == self
|
71
|
+
result = case adapter
|
72
|
+
when :redis
|
73
|
+
RedisAdapter
|
74
|
+
when Class
|
75
|
+
adapter
|
76
|
+
else
|
77
|
+
require "sidekiq/#{adapter}_adapter"
|
78
|
+
nil
|
79
|
+
end
|
80
|
+
@adapter = result if result
|
81
|
+
end
|
82
|
+
|
83
|
+
def create(options = {})
|
84
|
+
symbolized_options = options.transform_keys(&:to_sym)
|
85
|
+
|
86
|
+
if !symbolized_options[:url] && (u = determine_redis_provider)
|
87
|
+
symbolized_options[:url] = u
|
88
|
+
end
|
89
|
+
|
90
|
+
size = if symbolized_options[:size]
|
91
|
+
symbolized_options[:size]
|
92
|
+
elsif Sidekiq.server?
|
93
|
+
# Give ourselves plenty of connections. pool is lazy
|
94
|
+
# so we won't create them until we need them.
|
95
|
+
Sidekiq[:concurrency] + 5
|
96
|
+
elsif ENV["RAILS_MAX_THREADS"]
|
97
|
+
Integer(ENV["RAILS_MAX_THREADS"])
|
98
|
+
else
|
99
|
+
5
|
100
|
+
end
|
101
|
+
|
102
|
+
verify_sizing(size, Sidekiq[:concurrency]) if Sidekiq.server?
|
103
|
+
|
104
|
+
pool_timeout = symbolized_options[:pool_timeout] || 1
|
105
|
+
log_info(symbolized_options)
|
106
|
+
|
107
|
+
redis_config = adapter.new(symbolized_options)
|
108
|
+
ConnectionPool.new(timeout: pool_timeout, size: size) do
|
109
|
+
redis_config.new_client
|
110
|
+
end
|
111
|
+
end
|
112
|
+
|
113
|
+
private
|
114
|
+
|
115
|
+
# Sidekiq needs many concurrent Redis connections.
|
116
|
+
#
|
117
|
+
# We need a connection for each Processor.
|
118
|
+
# We need a connection for Pro's real-time change listener
|
119
|
+
# We need a connection to various features to call Redis every few seconds:
|
120
|
+
# - the process heartbeat.
|
121
|
+
# - enterprise's leader election
|
122
|
+
# - enterprise's cron support
|
123
|
+
def verify_sizing(size, concurrency)
|
124
|
+
raise ArgumentError, "Your Redis connection pool is too small for Sidekiq. Your pool has #{size} connections but must have at least #{concurrency + 2}" if size < (concurrency + 2)
|
125
|
+
end
|
93
126
|
|
94
127
|
def log_info(options)
|
95
128
|
redacted = "REDACTED"
|
@@ -110,9 +143,9 @@ module Sidekiq
|
|
110
143
|
sentinel[:password] = redacted if sentinel[:password]
|
111
144
|
end
|
112
145
|
if Sidekiq.server?
|
113
|
-
Sidekiq.logger.info("Booting Sidekiq #{Sidekiq::VERSION} with
|
146
|
+
Sidekiq.logger.info("Booting Sidekiq #{Sidekiq::VERSION} with #{adapter.name} options #{scrubbed_options}")
|
114
147
|
else
|
115
|
-
Sidekiq.logger.debug("#{Sidekiq::NAME} client with
|
148
|
+
Sidekiq.logger.debug("#{Sidekiq::NAME} client with #{adapter.name} options #{scrubbed_options}")
|
116
149
|
end
|
117
150
|
end
|
118
151
|
|
@@ -0,0 +1,29 @@
|
|
1
|
+
require "forwardable"
|
2
|
+
|
3
|
+
module Sidekiq
|
4
|
+
class RingBuffer
|
5
|
+
include Enumerable
|
6
|
+
extend Forwardable
|
7
|
+
def_delegators :@buf, :[], :each, :size
|
8
|
+
|
9
|
+
def initialize(size, default = 0)
|
10
|
+
@size = size
|
11
|
+
@buf = Array.new(size, default)
|
12
|
+
@index = 0
|
13
|
+
end
|
14
|
+
|
15
|
+
def <<(element)
|
16
|
+
@buf[@index % @size] = element
|
17
|
+
@index += 1
|
18
|
+
element
|
19
|
+
end
|
20
|
+
|
21
|
+
def buffer
|
22
|
+
@buf
|
23
|
+
end
|
24
|
+
|
25
|
+
def reset(default = 0)
|
26
|
+
@buf.fill(default)
|
27
|
+
end
|
28
|
+
end
|
29
|
+
end
|
data/lib/sidekiq/scheduled.rb
CHANGED
@@ -1,8 +1,8 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
3
|
require "sidekiq"
|
4
|
-
require "sidekiq/util"
|
5
4
|
require "sidekiq/api"
|
5
|
+
require "sidekiq/component"
|
6
6
|
|
7
7
|
module Sidekiq
|
8
8
|
module Scheduled
|
@@ -52,8 +52,8 @@ module Sidekiq
|
|
52
52
|
@lua_zpopbyscore_sha = raw_conn.script(:load, LUA_ZPOPBYSCORE)
|
53
53
|
end
|
54
54
|
|
55
|
-
conn.evalsha(@lua_zpopbyscore_sha, keys
|
56
|
-
rescue
|
55
|
+
conn.evalsha(@lua_zpopbyscore_sha, keys, argv)
|
56
|
+
rescue RedisConnection.adapter::CommandError => e
|
57
57
|
raise unless e.message.start_with?("NOSCRIPT")
|
58
58
|
|
59
59
|
@lua_zpopbyscore_sha = nil
|
@@ -67,12 +67,13 @@ module Sidekiq
|
|
67
67
|
# just pops the job back onto its original queue so the
|
68
68
|
# workers can pick it up like any other job.
|
69
69
|
class Poller
|
70
|
-
include
|
70
|
+
include Sidekiq::Component
|
71
71
|
|
72
72
|
INITIAL_WAIT = 10
|
73
73
|
|
74
|
-
def initialize
|
75
|
-
@
|
74
|
+
def initialize(options)
|
75
|
+
@config = options
|
76
|
+
@enq = (options[:scheduled_enq] || Sidekiq::Scheduled::Enq).new
|
76
77
|
@sleeper = ConnectionPool::TimedStack.new
|
77
78
|
@done = false
|
78
79
|
@thread = nil
|
@@ -100,7 +101,7 @@ module Sidekiq
|
|
100
101
|
enqueue
|
101
102
|
wait
|
102
103
|
end
|
103
|
-
|
104
|
+
logger.info("Scheduler exiting...")
|
104
105
|
}
|
105
106
|
end
|
106
107
|
|
@@ -171,14 +172,14 @@ module Sidekiq
|
|
171
172
|
#
|
172
173
|
# We only do this if poll_interval_average is unset (the default).
|
173
174
|
def poll_interval_average
|
174
|
-
|
175
|
+
@config[:poll_interval_average] ||= scaled_poll_interval
|
175
176
|
end
|
176
177
|
|
177
178
|
# Calculates an average poll interval based on the number of known Sidekiq processes.
|
178
179
|
# This minimizes a single point of failure by dispersing check-ins but without taxing
|
179
180
|
# Redis if you run many Sidekiq processes.
|
180
181
|
def scaled_poll_interval
|
181
|
-
process_count *
|
182
|
+
process_count * @config[:average_scheduled_poll_interval]
|
182
183
|
end
|
183
184
|
|
184
185
|
def process_count
|
@@ -197,7 +198,7 @@ module Sidekiq
|
|
197
198
|
# to give time for the heartbeat to register (if the poll interval is going to be calculated by the number
|
198
199
|
# of workers), and 5 random seconds to ensure they don't all hit Redis at the same time.
|
199
200
|
total = 0
|
200
|
-
total += INITIAL_WAIT unless
|
201
|
+
total += INITIAL_WAIT unless @config[:poll_interval_average]
|
201
202
|
total += (5 * rand)
|
202
203
|
|
203
204
|
@sleeper.pop(total)
|
@@ -4,7 +4,7 @@ require "sidekiq/testing"
|
|
4
4
|
|
5
5
|
##
|
6
6
|
# The Sidekiq inline infrastructure overrides perform_async so that it
|
7
|
-
# actually calls perform instead. This allows
|
7
|
+
# actually calls perform instead. This allows jobs to be run inline in a
|
8
8
|
# testing environment.
|
9
9
|
#
|
10
10
|
# This is similar to `Resque.inline = true` functionality.
|
@@ -15,8 +15,8 @@ require "sidekiq/testing"
|
|
15
15
|
#
|
16
16
|
# $external_variable = 0
|
17
17
|
#
|
18
|
-
# class
|
19
|
-
# include Sidekiq::
|
18
|
+
# class ExternalJob
|
19
|
+
# include Sidekiq::Job
|
20
20
|
#
|
21
21
|
# def perform
|
22
22
|
# $external_variable = 1
|
@@ -24,7 +24,7 @@ require "sidekiq/testing"
|
|
24
24
|
# end
|
25
25
|
#
|
26
26
|
# assert_equal 0, $external_variable
|
27
|
-
#
|
27
|
+
# ExternalJob.perform_async
|
28
28
|
# assert_equal 1, $external_variable
|
29
29
|
#
|
30
30
|
Sidekiq::Testing.inline!
|
data/lib/sidekiq/testing.rb
CHANGED
@@ -101,20 +101,20 @@ module Sidekiq
|
|
101
101
|
##
|
102
102
|
# The Queues class is only for testing the fake queue implementation.
|
103
103
|
# There are 2 data structures involved in tandem. This is due to the
|
104
|
-
# Rspec syntax of change(
|
104
|
+
# Rspec syntax of change(HardJob.jobs, :size). It keeps a reference
|
105
105
|
# to the array. Because the array was dervied from a filter of the total
|
106
106
|
# jobs enqueued, it appeared as though the array didn't change.
|
107
107
|
#
|
108
108
|
# To solve this, we'll keep 2 hashes containing the jobs. One with keys based
|
109
|
-
# on the queue, and another with keys of the
|
110
|
-
#
|
109
|
+
# on the queue, and another with keys of the job type, so the array for
|
110
|
+
# HardJob.jobs is a straight reference to a real array.
|
111
111
|
#
|
112
112
|
# Queue-based hash:
|
113
113
|
#
|
114
114
|
# {
|
115
115
|
# "default"=>[
|
116
116
|
# {
|
117
|
-
# "class"=>"TestTesting::
|
117
|
+
# "class"=>"TestTesting::HardJob",
|
118
118
|
# "args"=>[1, 2],
|
119
119
|
# "retry"=>true,
|
120
120
|
# "queue"=>"default",
|
@@ -124,12 +124,12 @@ module Sidekiq
|
|
124
124
|
# ]
|
125
125
|
# }
|
126
126
|
#
|
127
|
-
#
|
127
|
+
# Job-based hash:
|
128
128
|
#
|
129
129
|
# {
|
130
|
-
# "TestTesting::
|
130
|
+
# "TestTesting::HardJob"=>[
|
131
131
|
# {
|
132
|
-
# "class"=>"TestTesting::
|
132
|
+
# "class"=>"TestTesting::HardJob",
|
133
133
|
# "args"=>[1, 2],
|
134
134
|
# "retry"=>true,
|
135
135
|
# "queue"=>"default",
|
@@ -144,14 +144,14 @@ module Sidekiq
|
|
144
144
|
# require 'sidekiq/testing'
|
145
145
|
#
|
146
146
|
# assert_equal 0, Sidekiq::Queues["default"].size
|
147
|
-
#
|
147
|
+
# HardJob.perform_async(:something)
|
148
148
|
# assert_equal 1, Sidekiq::Queues["default"].size
|
149
149
|
# assert_equal :something, Sidekiq::Queues["default"].first['args'][0]
|
150
150
|
#
|
151
|
-
# You can also clear all
|
151
|
+
# You can also clear all jobs:
|
152
152
|
#
|
153
153
|
# assert_equal 0, Sidekiq::Queues["default"].size
|
154
|
-
#
|
154
|
+
# HardJob.perform_async(:something)
|
155
155
|
# Sidekiq::Queues.clear_all
|
156
156
|
# assert_equal 0, Sidekiq::Queues["default"].size
|
157
157
|
#
|
@@ -170,35 +170,36 @@ module Sidekiq
|
|
170
170
|
|
171
171
|
def push(queue, klass, job)
|
172
172
|
jobs_by_queue[queue] << job
|
173
|
-
|
173
|
+
jobs_by_class[klass] << job
|
174
174
|
end
|
175
175
|
|
176
176
|
def jobs_by_queue
|
177
177
|
@jobs_by_queue ||= Hash.new { |hash, key| hash[key] = [] }
|
178
178
|
end
|
179
179
|
|
180
|
-
def
|
181
|
-
@
|
180
|
+
def jobs_by_class
|
181
|
+
@jobs_by_class ||= Hash.new { |hash, key| hash[key] = [] }
|
182
182
|
end
|
183
|
+
alias_method :jobs_by_worker, :jobs_by_class
|
183
184
|
|
184
185
|
def delete_for(jid, queue, klass)
|
185
186
|
jobs_by_queue[queue.to_s].delete_if { |job| job["jid"] == jid }
|
186
|
-
|
187
|
+
jobs_by_class[klass].delete_if { |job| job["jid"] == jid }
|
187
188
|
end
|
188
189
|
|
189
190
|
def clear_for(queue, klass)
|
190
|
-
jobs_by_queue[queue].clear
|
191
|
-
|
191
|
+
jobs_by_queue[queue.to_s].clear
|
192
|
+
jobs_by_class[klass].clear
|
192
193
|
end
|
193
194
|
|
194
195
|
def clear_all
|
195
196
|
jobs_by_queue.clear
|
196
|
-
|
197
|
+
jobs_by_class.clear
|
197
198
|
end
|
198
199
|
end
|
199
200
|
end
|
200
201
|
|
201
|
-
module
|
202
|
+
module Job
|
202
203
|
##
|
203
204
|
# The Sidekiq testing infrastructure overrides perform_async
|
204
205
|
# so that it does not actually touch the network. Instead it
|
@@ -212,16 +213,16 @@ module Sidekiq
|
|
212
213
|
#
|
213
214
|
# require 'sidekiq/testing'
|
214
215
|
#
|
215
|
-
# assert_equal 0,
|
216
|
-
#
|
217
|
-
# assert_equal 1,
|
218
|
-
# assert_equal :something,
|
216
|
+
# assert_equal 0, HardJob.jobs.size
|
217
|
+
# HardJob.perform_async(:something)
|
218
|
+
# assert_equal 1, HardJob.jobs.size
|
219
|
+
# assert_equal :something, HardJob.jobs[0]['args'][0]
|
219
220
|
#
|
220
221
|
# assert_equal 0, Sidekiq::Extensions::DelayedMailer.jobs.size
|
221
222
|
# MyMailer.delay.send_welcome_email('foo@example.com')
|
222
223
|
# assert_equal 1, Sidekiq::Extensions::DelayedMailer.jobs.size
|
223
224
|
#
|
224
|
-
# You can also clear and drain all
|
225
|
+
# You can also clear and drain all job types:
|
225
226
|
#
|
226
227
|
# assert_equal 0, Sidekiq::Extensions::DelayedMailer.jobs.size
|
227
228
|
# assert_equal 0, Sidekiq::Extensions::DelayedModel.jobs.size
|
@@ -241,14 +242,14 @@ module Sidekiq
|
|
241
242
|
#
|
242
243
|
# RSpec.configure do |config|
|
243
244
|
# config.before(:each) do
|
244
|
-
# Sidekiq::
|
245
|
+
# Sidekiq::Job.clear_all
|
245
246
|
# end
|
246
247
|
# end
|
247
248
|
#
|
248
249
|
# or for acceptance testing, i.e. with cucumber:
|
249
250
|
#
|
250
251
|
# AfterStep do
|
251
|
-
# Sidekiq::
|
252
|
+
# Sidekiq::Job.drain_all
|
252
253
|
# end
|
253
254
|
#
|
254
255
|
# When I sign up as "foo@example.com"
|
@@ -262,7 +263,7 @@ module Sidekiq
|
|
262
263
|
|
263
264
|
# Jobs queued for this worker
|
264
265
|
def jobs
|
265
|
-
Queues.
|
266
|
+
Queues.jobs_by_class[to_s]
|
266
267
|
end
|
267
268
|
|
268
269
|
# Clear all jobs for this worker
|
@@ -288,11 +289,11 @@ module Sidekiq
|
|
288
289
|
end
|
289
290
|
|
290
291
|
def process_job(job)
|
291
|
-
|
292
|
-
|
293
|
-
|
294
|
-
Sidekiq::Testing.server_middleware.invoke(
|
295
|
-
execute_job(
|
292
|
+
inst = new
|
293
|
+
inst.jid = job["jid"]
|
294
|
+
inst.bid = job["bid"] if inst.respond_to?(:bid=)
|
295
|
+
Sidekiq::Testing.server_middleware.invoke(inst, job, job["queue"]) do
|
296
|
+
execute_job(inst, job["args"])
|
296
297
|
end
|
297
298
|
end
|
298
299
|
|
@@ -306,18 +307,18 @@ module Sidekiq
|
|
306
307
|
Queues.jobs_by_queue.values.flatten
|
307
308
|
end
|
308
309
|
|
309
|
-
# Clear all queued jobs
|
310
|
+
# Clear all queued jobs
|
310
311
|
def clear_all
|
311
312
|
Queues.clear_all
|
312
313
|
end
|
313
314
|
|
314
|
-
# Drain all queued jobs
|
315
|
+
# Drain (execute) all queued jobs
|
315
316
|
def drain_all
|
316
317
|
while jobs.any?
|
317
|
-
|
318
|
+
job_classes = jobs.map { |job| job["class"] }.uniq
|
318
319
|
|
319
|
-
|
320
|
-
Sidekiq::Testing.constantize(
|
320
|
+
job_classes.each do |job_class|
|
321
|
+
Sidekiq::Testing.constantize(job_class).drain
|
321
322
|
end
|
322
323
|
end
|
323
324
|
end
|
@@ -0,0 +1,45 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require "securerandom"
|
4
|
+
require "sidekiq/client"
|
5
|
+
|
6
|
+
module Sidekiq
|
7
|
+
class TransactionAwareClient
|
8
|
+
def initialize(redis_pool)
|
9
|
+
@redis_client = Client.new(redis_pool)
|
10
|
+
end
|
11
|
+
|
12
|
+
def push(item)
|
13
|
+
# pre-allocate the JID so we can return it immediately and
|
14
|
+
# save it to the database as part of the transaction.
|
15
|
+
item["jid"] ||= SecureRandom.hex(12)
|
16
|
+
AfterCommitEverywhere.after_commit { @redis_client.push(item) }
|
17
|
+
item["jid"]
|
18
|
+
end
|
19
|
+
|
20
|
+
##
|
21
|
+
# We don't provide transactionality for push_bulk because we don't want
|
22
|
+
# to hold potentially hundreds of thousands of job records in memory due to
|
23
|
+
# a long running enqueue process.
|
24
|
+
def push_bulk(items)
|
25
|
+
@redis_client.push_bulk(items)
|
26
|
+
end
|
27
|
+
end
|
28
|
+
end
|
29
|
+
|
30
|
+
##
|
31
|
+
# Use `Sidekiq.transactional_push!` in your sidekiq.rb initializer
|
32
|
+
module Sidekiq
|
33
|
+
def self.transactional_push!
|
34
|
+
begin
|
35
|
+
require "after_commit_everywhere"
|
36
|
+
rescue LoadError
|
37
|
+
Sidekiq.logger.error("You need to add after_commit_everywhere to your Gemfile to use Sidekiq's transactional client")
|
38
|
+
raise
|
39
|
+
end
|
40
|
+
|
41
|
+
default_job_options["client_class"] = Sidekiq::TransactionAwareClient
|
42
|
+
Sidekiq::JobUtil::TRANSIENT_ATTRIBUTES << "client_class"
|
43
|
+
true
|
44
|
+
end
|
45
|
+
end
|
data/lib/sidekiq/version.rb
CHANGED