sr-sidekiq 4.1.6
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +7 -0
- data/.gitignore +12 -0
- data/3.0-Upgrade.md +70 -0
- data/4.0-Upgrade.md +50 -0
- data/COMM-LICENSE (sidekiq) +95 -0
- data/Changes.md +1241 -0
- data/Ent-Changes.md +112 -0
- data/Gemfile +29 -0
- data/LICENSE (sidekiq) +9 -0
- data/LICENSE (sr-sidekiq) +5 -0
- data/Pro-2.0-Upgrade.md +138 -0
- data/Pro-3.0-Upgrade.md +44 -0
- data/Pro-Changes.md +539 -0
- data/README.md +8 -0
- data/Rakefile +9 -0
- data/bin/sidekiq +18 -0
- data/bin/sidekiqctl +99 -0
- data/bin/sidekiqload +167 -0
- data/code_of_conduct.md +50 -0
- data/lib/generators/sidekiq/templates/worker.rb.erb +9 -0
- data/lib/generators/sidekiq/templates/worker_spec.rb.erb +6 -0
- data/lib/generators/sidekiq/templates/worker_test.rb.erb +8 -0
- data/lib/generators/sidekiq/worker_generator.rb +49 -0
- data/lib/sidekiq.rb +237 -0
- data/lib/sidekiq/api.rb +844 -0
- data/lib/sidekiq/cli.rb +389 -0
- data/lib/sidekiq/client.rb +260 -0
- data/lib/sidekiq/core_ext.rb +106 -0
- data/lib/sidekiq/exception_handler.rb +31 -0
- data/lib/sidekiq/extensions/action_mailer.rb +57 -0
- data/lib/sidekiq/extensions/active_record.rb +40 -0
- data/lib/sidekiq/extensions/class_methods.rb +40 -0
- data/lib/sidekiq/extensions/generic_proxy.rb +25 -0
- data/lib/sidekiq/fetch.rb +81 -0
- data/lib/sidekiq/launcher.rb +160 -0
- data/lib/sidekiq/logging.rb +106 -0
- data/lib/sidekiq/manager.rb +137 -0
- data/lib/sidekiq/middleware/chain.rb +150 -0
- data/lib/sidekiq/middleware/i18n.rb +42 -0
- data/lib/sidekiq/middleware/server/active_record.rb +13 -0
- data/lib/sidekiq/middleware/server/logging.rb +40 -0
- data/lib/sidekiq/middleware/server/retry_jobs.rb +205 -0
- data/lib/sidekiq/paginator.rb +43 -0
- data/lib/sidekiq/processor.rb +186 -0
- data/lib/sidekiq/rails.rb +39 -0
- data/lib/sidekiq/redis_connection.rb +97 -0
- data/lib/sidekiq/scheduled.rb +146 -0
- data/lib/sidekiq/testing.rb +316 -0
- data/lib/sidekiq/testing/inline.rb +29 -0
- data/lib/sidekiq/util.rb +62 -0
- data/lib/sidekiq/version.rb +4 -0
- data/lib/sidekiq/web.rb +278 -0
- data/lib/sidekiq/web_helpers.rb +255 -0
- data/lib/sidekiq/worker.rb +121 -0
- data/sidekiq.gemspec +26 -0
- data/sr-sidekiq-4.1.3.gem +0 -0
- data/sr-sidekiq-4.1.4.gem +0 -0
- data/sr-sidekiq-4.1.5.gem +0 -0
- data/test/config.yml +9 -0
- data/test/env_based_config.yml +11 -0
- data/test/fake_env.rb +1 -0
- data/test/fixtures/en.yml +2 -0
- data/test/helper.rb +75 -0
- data/test/test_actors.rb +138 -0
- data/test/test_api.rb +528 -0
- data/test/test_cli.rb +406 -0
- data/test/test_client.rb +262 -0
- data/test/test_exception_handler.rb +56 -0
- data/test/test_extensions.rb +127 -0
- data/test/test_fetch.rb +50 -0
- data/test/test_launcher.rb +85 -0
- data/test/test_logging.rb +35 -0
- data/test/test_manager.rb +50 -0
- data/test/test_middleware.rb +158 -0
- data/test/test_processor.rb +201 -0
- data/test/test_rails.rb +22 -0
- data/test/test_redis_connection.rb +127 -0
- data/test/test_retry.rb +326 -0
- data/test/test_retry_exhausted.rb +149 -0
- data/test/test_scheduled.rb +115 -0
- data/test/test_scheduling.rb +50 -0
- data/test/test_sidekiq.rb +107 -0
- data/test/test_testing.rb +143 -0
- data/test/test_testing_fake.rb +357 -0
- data/test/test_testing_inline.rb +94 -0
- data/test/test_util.rb +13 -0
- data/test/test_web.rb +614 -0
- data/test/test_web_helpers.rb +54 -0
- data/web/assets/images/bootstrap/glyphicons-halflings-white.png +0 -0
- data/web/assets/images/bootstrap/glyphicons-halflings.png +0 -0
- data/web/assets/images/favicon.ico +0 -0
- data/web/assets/images/logo.png +0 -0
- data/web/assets/images/status-sd8051fd480.png +0 -0
- data/web/assets/images/status/active.png +0 -0
- data/web/assets/images/status/idle.png +0 -0
- data/web/assets/javascripts/application.js +88 -0
- data/web/assets/javascripts/dashboard.js +300 -0
- data/web/assets/javascripts/locales/README.md +27 -0
- data/web/assets/javascripts/locales/jquery.timeago.ar.js +96 -0
- data/web/assets/javascripts/locales/jquery.timeago.bg.js +18 -0
- data/web/assets/javascripts/locales/jquery.timeago.bs.js +49 -0
- data/web/assets/javascripts/locales/jquery.timeago.ca.js +18 -0
- data/web/assets/javascripts/locales/jquery.timeago.cs.js +18 -0
- data/web/assets/javascripts/locales/jquery.timeago.cy.js +20 -0
- data/web/assets/javascripts/locales/jquery.timeago.da.js +18 -0
- data/web/assets/javascripts/locales/jquery.timeago.de.js +18 -0
- data/web/assets/javascripts/locales/jquery.timeago.el.js +18 -0
- data/web/assets/javascripts/locales/jquery.timeago.en-short.js +20 -0
- data/web/assets/javascripts/locales/jquery.timeago.en.js +20 -0
- data/web/assets/javascripts/locales/jquery.timeago.es.js +18 -0
- data/web/assets/javascripts/locales/jquery.timeago.et.js +18 -0
- data/web/assets/javascripts/locales/jquery.timeago.fa.js +22 -0
- data/web/assets/javascripts/locales/jquery.timeago.fi.js +28 -0
- data/web/assets/javascripts/locales/jquery.timeago.fr-short.js +16 -0
- data/web/assets/javascripts/locales/jquery.timeago.fr.js +17 -0
- data/web/assets/javascripts/locales/jquery.timeago.he.js +18 -0
- data/web/assets/javascripts/locales/jquery.timeago.hr.js +49 -0
- data/web/assets/javascripts/locales/jquery.timeago.hu.js +18 -0
- data/web/assets/javascripts/locales/jquery.timeago.hy.js +18 -0
- data/web/assets/javascripts/locales/jquery.timeago.id.js +18 -0
- data/web/assets/javascripts/locales/jquery.timeago.it.js +16 -0
- data/web/assets/javascripts/locales/jquery.timeago.ja.js +19 -0
- data/web/assets/javascripts/locales/jquery.timeago.ko.js +17 -0
- data/web/assets/javascripts/locales/jquery.timeago.lt.js +20 -0
- data/web/assets/javascripts/locales/jquery.timeago.mk.js +20 -0
- data/web/assets/javascripts/locales/jquery.timeago.nl.js +20 -0
- data/web/assets/javascripts/locales/jquery.timeago.no.js +18 -0
- data/web/assets/javascripts/locales/jquery.timeago.pl.js +31 -0
- data/web/assets/javascripts/locales/jquery.timeago.pt-br.js +16 -0
- data/web/assets/javascripts/locales/jquery.timeago.pt.js +16 -0
- data/web/assets/javascripts/locales/jquery.timeago.ro.js +18 -0
- data/web/assets/javascripts/locales/jquery.timeago.rs.js +49 -0
- data/web/assets/javascripts/locales/jquery.timeago.ru.js +34 -0
- data/web/assets/javascripts/locales/jquery.timeago.sk.js +18 -0
- data/web/assets/javascripts/locales/jquery.timeago.sl.js +44 -0
- data/web/assets/javascripts/locales/jquery.timeago.sv.js +18 -0
- data/web/assets/javascripts/locales/jquery.timeago.th.js +20 -0
- data/web/assets/javascripts/locales/jquery.timeago.tr.js +16 -0
- data/web/assets/javascripts/locales/jquery.timeago.uk.js +34 -0
- data/web/assets/javascripts/locales/jquery.timeago.uz.js +19 -0
- data/web/assets/javascripts/locales/jquery.timeago.zh-cn.js +20 -0
- data/web/assets/javascripts/locales/jquery.timeago.zh-tw.js +20 -0
- data/web/assets/stylesheets/application.css +754 -0
- data/web/assets/stylesheets/bootstrap.css +9 -0
- data/web/locales/cs.yml +78 -0
- data/web/locales/da.yml +68 -0
- data/web/locales/de.yml +69 -0
- data/web/locales/el.yml +68 -0
- data/web/locales/en.yml +79 -0
- data/web/locales/es.yml +69 -0
- data/web/locales/fr.yml +78 -0
- data/web/locales/hi.yml +75 -0
- data/web/locales/it.yml +69 -0
- data/web/locales/ja.yml +78 -0
- data/web/locales/ko.yml +68 -0
- data/web/locales/nb.yml +77 -0
- data/web/locales/nl.yml +68 -0
- data/web/locales/pl.yml +59 -0
- data/web/locales/pt-br.yml +68 -0
- data/web/locales/pt.yml +67 -0
- data/web/locales/ru.yml +78 -0
- data/web/locales/sv.yml +68 -0
- data/web/locales/ta.yml +75 -0
- data/web/locales/uk.yml +76 -0
- data/web/locales/zh-cn.yml +68 -0
- data/web/locales/zh-tw.yml +68 -0
- data/web/views/_footer.erb +17 -0
- data/web/views/_job_info.erb +88 -0
- data/web/views/_nav.erb +66 -0
- data/web/views/_paging.erb +23 -0
- data/web/views/_poll_js.erb +5 -0
- data/web/views/_poll_link.erb +7 -0
- data/web/views/_status.erb +4 -0
- data/web/views/_summary.erb +40 -0
- data/web/views/busy.erb +94 -0
- data/web/views/dashboard.erb +75 -0
- data/web/views/dead.erb +34 -0
- data/web/views/layout.erb +32 -0
- data/web/views/morgue.erb +71 -0
- data/web/views/queue.erb +45 -0
- data/web/views/queues.erb +28 -0
- data/web/views/retries.erb +74 -0
- data/web/views/retry.erb +34 -0
- data/web/views/scheduled.erb +54 -0
- data/web/views/scheduled_job_info.erb +8 -0
- metadata +408 -0
@@ -0,0 +1,39 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
module Sidekiq
|
3
|
+
def self.hook_rails!
|
4
|
+
return if defined?(@delay_removed)
|
5
|
+
|
6
|
+
ActiveSupport.on_load(:active_record) do
|
7
|
+
include Sidekiq::Extensions::ActiveRecord
|
8
|
+
end
|
9
|
+
|
10
|
+
ActiveSupport.on_load(:action_mailer) do
|
11
|
+
extend Sidekiq::Extensions::ActionMailer
|
12
|
+
end
|
13
|
+
|
14
|
+
Module.__send__(:include, Sidekiq::Extensions::Klass)
|
15
|
+
end
|
16
|
+
|
17
|
+
# Removes the generic aliases which MAY clash with names of already
|
18
|
+
# created methods by other applications. The methods `sidekiq_delay`,
|
19
|
+
# `sidekiq_delay_for` and `sidekiq_delay_until` can be used instead.
|
20
|
+
def self.remove_delay!
|
21
|
+
@delay_removed = true
|
22
|
+
|
23
|
+
[Extensions::ActiveRecord,
|
24
|
+
Extensions::ActionMailer,
|
25
|
+
Extensions::Klass].each do |mod|
|
26
|
+
mod.module_eval do
|
27
|
+
remove_method :delay if respond_to?(:delay)
|
28
|
+
remove_method :delay_for if respond_to?(:delay_for)
|
29
|
+
remove_method :delay_until if respond_to?(:delay_until)
|
30
|
+
end
|
31
|
+
end
|
32
|
+
end
|
33
|
+
|
34
|
+
class Rails < ::Rails::Engine
|
35
|
+
initializer 'sidekiq' do
|
36
|
+
Sidekiq.hook_rails!
|
37
|
+
end
|
38
|
+
end if defined?(::Rails)
|
39
|
+
end
|
@@ -0,0 +1,97 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
require 'connection_pool'
|
3
|
+
require 'redis'
|
4
|
+
require 'uri'
|
5
|
+
|
6
|
+
module Sidekiq
|
7
|
+
class RedisConnection
|
8
|
+
class << self
|
9
|
+
|
10
|
+
def create(options={})
|
11
|
+
options[:url] ||= determine_redis_provider
|
12
|
+
|
13
|
+
size = options[:size] || (Sidekiq.server? ? (Sidekiq.options[:concurrency] + 5) : 5)
|
14
|
+
|
15
|
+
verify_sizing(size, Sidekiq.options[:concurrency]) if Sidekiq.server?
|
16
|
+
|
17
|
+
pool_timeout = options[:pool_timeout] || 1
|
18
|
+
log_info(options)
|
19
|
+
|
20
|
+
ConnectionPool.new(:timeout => pool_timeout, :size => size) do
|
21
|
+
build_client(options)
|
22
|
+
end
|
23
|
+
end
|
24
|
+
|
25
|
+
private
|
26
|
+
|
27
|
+
# Sidekiq needs a lot of concurrent Redis connections.
|
28
|
+
#
|
29
|
+
# We need a connection for each Processor.
|
30
|
+
# We need a connection for Pro's real-time change listener
|
31
|
+
# We need a connection to various features to call Redis every few seconds:
|
32
|
+
# - the process heartbeat.
|
33
|
+
# - enterprise's leader election
|
34
|
+
# - enterprise's cron support
|
35
|
+
def verify_sizing(size, concurrency)
|
36
|
+
raise ArgumentError, "Your Redis connection pool is too small for Sidekiq to work, your pool has #{size} connections but really needs to have at least #{concurrency + 2}" if size <= concurrency
|
37
|
+
end
|
38
|
+
|
39
|
+
def build_client(options)
|
40
|
+
namespace = options[:namespace]
|
41
|
+
|
42
|
+
client = Redis.new client_opts(options)
|
43
|
+
if namespace
|
44
|
+
begin
|
45
|
+
require 'redis/namespace'
|
46
|
+
Redis::Namespace.new(namespace, :redis => client)
|
47
|
+
rescue LoadError
|
48
|
+
Sidekiq.logger.error("Your Redis configuration use the namespace '#{namespace}' but the redis-namespace gem not included in Gemfile." \
|
49
|
+
"Add the gem to your Gemfile in case you would like to keep using a namespace, otherwise remove the namespace parameter.")
|
50
|
+
exit(-127)
|
51
|
+
end
|
52
|
+
else
|
53
|
+
client
|
54
|
+
end
|
55
|
+
end
|
56
|
+
|
57
|
+
def client_opts(options)
|
58
|
+
opts = options.dup
|
59
|
+
if opts[:namespace]
|
60
|
+
opts.delete(:namespace)
|
61
|
+
end
|
62
|
+
|
63
|
+
if opts[:network_timeout]
|
64
|
+
opts[:timeout] = opts[:network_timeout]
|
65
|
+
opts.delete(:network_timeout)
|
66
|
+
end
|
67
|
+
|
68
|
+
opts[:driver] = opts[:driver] || 'ruby'
|
69
|
+
|
70
|
+
opts
|
71
|
+
end
|
72
|
+
|
73
|
+
def log_info(options)
|
74
|
+
# Don't log Redis AUTH password
|
75
|
+
redacted = "REDACTED"
|
76
|
+
scrubbed_options = options.dup
|
77
|
+
if scrubbed_options[:url] && (uri = URI.parse(scrubbed_options[:url])) && uri.password
|
78
|
+
uri.password = redacted
|
79
|
+
scrubbed_options[:url] = uri.to_s
|
80
|
+
end
|
81
|
+
if scrubbed_options[:password]
|
82
|
+
scrubbed_options[:password] = redacted
|
83
|
+
end
|
84
|
+
if Sidekiq.server?
|
85
|
+
Sidekiq.logger.info("Booting Sidekiq #{Sidekiq::VERSION} with redis options #{scrubbed_options}")
|
86
|
+
else
|
87
|
+
Sidekiq.logger.debug("#{Sidekiq::NAME} client with redis options #{scrubbed_options}")
|
88
|
+
end
|
89
|
+
end
|
90
|
+
|
91
|
+
def determine_redis_provider
|
92
|
+
ENV[ENV['REDIS_PROVIDER'] || 'REDIS_URL']
|
93
|
+
end
|
94
|
+
|
95
|
+
end
|
96
|
+
end
|
97
|
+
end
|
@@ -0,0 +1,146 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
require 'sidekiq'
|
3
|
+
require 'sidekiq/util'
|
4
|
+
require 'sidekiq/api'
|
5
|
+
|
6
|
+
module Sidekiq
|
7
|
+
module Scheduled
|
8
|
+
SETS = %w(retry schedule)
|
9
|
+
|
10
|
+
class Enq
|
11
|
+
def enqueue_jobs(now=Time.now.to_f.to_s, sorted_sets=SETS)
|
12
|
+
# A job's "score" in Redis is the time at which it should be processed.
|
13
|
+
# Just check Redis for the set of jobs with a timestamp before now.
|
14
|
+
Sidekiq.redis do |conn|
|
15
|
+
sorted_sets.each do |sorted_set|
|
16
|
+
# Get the next item in the queue if it's score (time to execute) is <= now.
|
17
|
+
# We need to go through the list one at a time to reduce the risk of something
|
18
|
+
# going wrong between the time jobs are popped from the scheduled queue and when
|
19
|
+
# they are pushed onto a work queue and losing the jobs.
|
20
|
+
while job = conn.zrangebyscore(sorted_set, '-inf'.freeze, now, :limit => [0, 1]).first do
|
21
|
+
|
22
|
+
# Pop item off the queue and add it to the work queue. If the job can't be popped from
|
23
|
+
# the queue, it's because another process already popped it so we can move on to the
|
24
|
+
# next one.
|
25
|
+
if conn.zrem(sorted_set, job)
|
26
|
+
Sidekiq::Client.push(Sidekiq.load_json(job))
|
27
|
+
Sidekiq::Logging.logger.debug { "enqueued #{sorted_set}: #{job}" }
|
28
|
+
end
|
29
|
+
end
|
30
|
+
end
|
31
|
+
end
|
32
|
+
end
|
33
|
+
end
|
34
|
+
|
35
|
+
##
|
36
|
+
# The Poller checks Redis every N seconds for jobs in the retry or scheduled
|
37
|
+
# set have passed their timestamp and should be enqueued. If so, it
|
38
|
+
# just pops the job back onto its original queue so the
|
39
|
+
# workers can pick it up like any other job.
|
40
|
+
class Poller
|
41
|
+
include Util
|
42
|
+
|
43
|
+
INITIAL_WAIT = 10
|
44
|
+
|
45
|
+
def initialize
|
46
|
+
@enq = (Sidekiq.options[:scheduled_enq] || Sidekiq::Scheduled::Enq).new
|
47
|
+
@sleeper = ConnectionPool::TimedStack.new
|
48
|
+
@done = false
|
49
|
+
end
|
50
|
+
|
51
|
+
# Shut down this instance, will pause until the thread is dead.
|
52
|
+
def terminate
|
53
|
+
@done = true
|
54
|
+
if @thread
|
55
|
+
t = @thread
|
56
|
+
@thread = nil
|
57
|
+
@sleeper << 0
|
58
|
+
t.value
|
59
|
+
end
|
60
|
+
end
|
61
|
+
|
62
|
+
def start
|
63
|
+
@thread ||= safe_thread("scheduler") do
|
64
|
+
initial_wait
|
65
|
+
|
66
|
+
while !@done
|
67
|
+
enqueue
|
68
|
+
wait
|
69
|
+
end
|
70
|
+
Sidekiq.logger.info("Scheduler exiting...")
|
71
|
+
end
|
72
|
+
end
|
73
|
+
|
74
|
+
def enqueue
|
75
|
+
begin
|
76
|
+
@enq.enqueue_jobs
|
77
|
+
rescue => ex
|
78
|
+
# Most likely a problem with redis networking.
|
79
|
+
# Punt and try again at the next interval
|
80
|
+
logger.error ex.message
|
81
|
+
ex.backtrace.each do |bt|
|
82
|
+
logger.error(bt)
|
83
|
+
end
|
84
|
+
end
|
85
|
+
end
|
86
|
+
|
87
|
+
private
|
88
|
+
|
89
|
+
def wait
|
90
|
+
@sleeper.pop(random_poll_interval)
|
91
|
+
rescue Timeout::Error
|
92
|
+
# expected
|
93
|
+
rescue => ex
|
94
|
+
# if poll_interval_average hasn't been calculated yet, we can
|
95
|
+
# raise an error trying to reach Redis.
|
96
|
+
logger.error ex.message
|
97
|
+
logger.error ex.backtrace.first
|
98
|
+
sleep 5
|
99
|
+
end
|
100
|
+
|
101
|
+
# Calculates a random interval that is ±50% the desired average.
|
102
|
+
def random_poll_interval
|
103
|
+
poll_interval_average * rand + poll_interval_average.to_f / 2
|
104
|
+
end
|
105
|
+
|
106
|
+
# We do our best to tune the poll interval to the size of the active Sidekiq
|
107
|
+
# cluster. If you have 30 processes and poll every 15 seconds, that means one
|
108
|
+
# Sidekiq is checking Redis every 0.5 seconds - way too often for most people
|
109
|
+
# and really bad if the retry or scheduled sets are large.
|
110
|
+
#
|
111
|
+
# Instead try to avoid polling more than once every 15 seconds. If you have
|
112
|
+
# 30 Sidekiq processes, we'll poll every 30 * 15 or 450 seconds.
|
113
|
+
# To keep things statistically random, we'll sleep a random amount between
|
114
|
+
# 225 and 675 seconds for each poll or 450 seconds on average. Otherwise restarting
|
115
|
+
# all your Sidekiq processes at the same time will lead to them all polling at
|
116
|
+
# the same time: the thundering herd problem.
|
117
|
+
#
|
118
|
+
# We only do this if poll_interval_average is unset (the default).
|
119
|
+
def poll_interval_average
|
120
|
+
Sidekiq.options[:poll_interval_average] ||= scaled_poll_interval
|
121
|
+
end
|
122
|
+
|
123
|
+
# Calculates an average poll interval based on the number of known Sidekiq processes.
|
124
|
+
# This minimizes a single point of failure by dispersing check-ins but without taxing
|
125
|
+
# Redis if you run many Sidekiq processes.
|
126
|
+
def scaled_poll_interval
|
127
|
+
pcount = Sidekiq::ProcessSet.new.size
|
128
|
+
pcount = 1 if pcount == 0
|
129
|
+
pcount * Sidekiq.options[:average_scheduled_poll_interval]
|
130
|
+
end
|
131
|
+
|
132
|
+
def initial_wait
|
133
|
+
# Have all processes sleep between 5-15 seconds. 10 seconds
|
134
|
+
# to give time for the heartbeat to register (if the poll interval is going to be calculated by the number
|
135
|
+
# of workers), and 5 random seconds to ensure they don't all hit Redis at the same time.
|
136
|
+
total = 0
|
137
|
+
total += INITIAL_WAIT unless Sidekiq.options[:poll_interval_average]
|
138
|
+
total += (5 * rand)
|
139
|
+
|
140
|
+
@sleeper.pop(total)
|
141
|
+
rescue Timeout::Error
|
142
|
+
end
|
143
|
+
|
144
|
+
end
|
145
|
+
end
|
146
|
+
end
|
@@ -0,0 +1,316 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
require 'securerandom'
|
3
|
+
require 'sidekiq'
|
4
|
+
|
5
|
+
module Sidekiq
|
6
|
+
|
7
|
+
class Testing
|
8
|
+
class << self
|
9
|
+
attr_accessor :__test_mode
|
10
|
+
|
11
|
+
def __set_test_mode(mode)
|
12
|
+
if block_given?
|
13
|
+
current_mode = self.__test_mode
|
14
|
+
begin
|
15
|
+
self.__test_mode = mode
|
16
|
+
yield
|
17
|
+
ensure
|
18
|
+
self.__test_mode = current_mode
|
19
|
+
end
|
20
|
+
else
|
21
|
+
self.__test_mode = mode
|
22
|
+
end
|
23
|
+
end
|
24
|
+
|
25
|
+
def disable!(&block)
|
26
|
+
__set_test_mode(:disable, &block)
|
27
|
+
end
|
28
|
+
|
29
|
+
def fake!(&block)
|
30
|
+
__set_test_mode(:fake, &block)
|
31
|
+
end
|
32
|
+
|
33
|
+
def inline!(&block)
|
34
|
+
__set_test_mode(:inline, &block)
|
35
|
+
end
|
36
|
+
|
37
|
+
def enabled?
|
38
|
+
self.__test_mode != :disable
|
39
|
+
end
|
40
|
+
|
41
|
+
def disabled?
|
42
|
+
self.__test_mode == :disable
|
43
|
+
end
|
44
|
+
|
45
|
+
def fake?
|
46
|
+
self.__test_mode == :fake
|
47
|
+
end
|
48
|
+
|
49
|
+
def inline?
|
50
|
+
self.__test_mode == :inline
|
51
|
+
end
|
52
|
+
|
53
|
+
def server_middleware
|
54
|
+
@server_chain ||= Middleware::Chain.new
|
55
|
+
yield @server_chain if block_given?
|
56
|
+
@server_chain
|
57
|
+
end
|
58
|
+
end
|
59
|
+
end
|
60
|
+
|
61
|
+
# Default to fake testing to keep old behavior
|
62
|
+
Sidekiq::Testing.fake!
|
63
|
+
|
64
|
+
class EmptyQueueError < RuntimeError; end
|
65
|
+
|
66
|
+
class Client
|
67
|
+
alias_method :raw_push_real, :raw_push
|
68
|
+
|
69
|
+
def raw_push(payloads)
|
70
|
+
if Sidekiq::Testing.fake?
|
71
|
+
payloads.each do |job|
|
72
|
+
Queues.push(job['queue'], job['class'], Sidekiq.load_json(Sidekiq.dump_json(job)))
|
73
|
+
end
|
74
|
+
true
|
75
|
+
elsif Sidekiq::Testing.inline?
|
76
|
+
payloads.each do |job|
|
77
|
+
klass = job['class'].constantize
|
78
|
+
job['id'] ||= SecureRandom.hex(12)
|
79
|
+
job_hash = Sidekiq.load_json(Sidekiq.dump_json(job))
|
80
|
+
klass.process_job(job_hash)
|
81
|
+
end
|
82
|
+
true
|
83
|
+
else
|
84
|
+
raw_push_real(payloads)
|
85
|
+
end
|
86
|
+
end
|
87
|
+
end
|
88
|
+
|
89
|
+
module Queues
|
90
|
+
##
|
91
|
+
# The Queues class is only for testing the fake queue implementation.
|
92
|
+
# There are 2 data structures involved in tandem. This is due to the
|
93
|
+
# Rspec syntax of change(QueueWorker.jobs, :size). It keeps a reference
|
94
|
+
# to the array. Because the array was dervied from a filter of the total
|
95
|
+
# jobs enqueued, it appeared as though the array didn't change.
|
96
|
+
#
|
97
|
+
# To solve this, we'll keep 2 hashes containing the jobs. One with keys based
|
98
|
+
# on the queue, and another with keys of the worker names, so the array for
|
99
|
+
# QueueWorker.jobs is a straight reference to a real array.
|
100
|
+
#
|
101
|
+
# Queue-based hash:
|
102
|
+
#
|
103
|
+
# {
|
104
|
+
# "default"=>[
|
105
|
+
# {
|
106
|
+
# "class"=>"TestTesting::QueueWorker",
|
107
|
+
# "args"=>[1, 2],
|
108
|
+
# "retry"=>true,
|
109
|
+
# "queue"=>"default",
|
110
|
+
# "jid"=>"abc5b065c5c4b27fc1102833",
|
111
|
+
# "created_at"=>1447445554.419934
|
112
|
+
# }
|
113
|
+
# ]
|
114
|
+
# }
|
115
|
+
#
|
116
|
+
# Worker-based hash:
|
117
|
+
#
|
118
|
+
# {
|
119
|
+
# "TestTesting::QueueWorker"=>[
|
120
|
+
# {
|
121
|
+
# "class"=>"TestTesting::QueueWorker",
|
122
|
+
# "args"=>[1, 2],
|
123
|
+
# "retry"=>true,
|
124
|
+
# "queue"=>"default",
|
125
|
+
# "jid"=>"abc5b065c5c4b27fc1102833",
|
126
|
+
# "created_at"=>1447445554.419934
|
127
|
+
# }
|
128
|
+
# ]
|
129
|
+
# }
|
130
|
+
#
|
131
|
+
# Example:
|
132
|
+
#
|
133
|
+
# require 'sidekiq/testing'
|
134
|
+
#
|
135
|
+
# assert_equal 0, Sidekiq::Queues["default"].size
|
136
|
+
# HardWorker.perform_async(:something)
|
137
|
+
# assert_equal 1, Sidekiq::Queues["default"].size
|
138
|
+
# assert_equal :something, Sidekiq::Queues["default"].first['args'][0]
|
139
|
+
#
|
140
|
+
# You can also clear all workers' jobs:
|
141
|
+
#
|
142
|
+
# assert_equal 0, Sidekiq::Queues["default"].size
|
143
|
+
# HardWorker.perform_async(:something)
|
144
|
+
# Sidekiq::Queues.clear_all
|
145
|
+
# assert_equal 0, Sidekiq::Queues["default"].size
|
146
|
+
#
|
147
|
+
# This can be useful to make sure jobs don't linger between tests:
|
148
|
+
#
|
149
|
+
# RSpec.configure do |config|
|
150
|
+
# config.before(:each) do
|
151
|
+
# Sidekiq::Queues.clear_all
|
152
|
+
# end
|
153
|
+
# end
|
154
|
+
#
|
155
|
+
class << self
|
156
|
+
def [](queue)
|
157
|
+
jobs_by_queue[queue]
|
158
|
+
end
|
159
|
+
|
160
|
+
def push(queue, klass, job)
|
161
|
+
jobs_by_queue[queue] << job
|
162
|
+
jobs_by_worker[klass] << job
|
163
|
+
end
|
164
|
+
|
165
|
+
def jobs_by_queue
|
166
|
+
@jobs_by_queue ||= Hash.new { |hash, key| hash[key] = [] }
|
167
|
+
end
|
168
|
+
|
169
|
+
def jobs_by_worker
|
170
|
+
@jobs_by_worker ||= Hash.new { |hash, key| hash[key] = [] }
|
171
|
+
end
|
172
|
+
|
173
|
+
def delete_for(jid, queue, klass)
|
174
|
+
jobs_by_queue[queue.to_s].delete_if { |job| job["jid"] == jid }
|
175
|
+
jobs_by_worker[klass].delete_if { |job| job["jid"] == jid }
|
176
|
+
end
|
177
|
+
|
178
|
+
def clear_for(queue, klass)
|
179
|
+
jobs_by_queue[queue].clear
|
180
|
+
jobs_by_worker[klass].clear
|
181
|
+
end
|
182
|
+
|
183
|
+
def clear_all
|
184
|
+
jobs_by_queue.clear
|
185
|
+
jobs_by_worker.clear
|
186
|
+
end
|
187
|
+
end
|
188
|
+
end
|
189
|
+
|
190
|
+
module Worker
|
191
|
+
##
|
192
|
+
# The Sidekiq testing infrastructure overrides perform_async
|
193
|
+
# so that it does not actually touch the network. Instead it
|
194
|
+
# stores the asynchronous jobs in a per-class array so that
|
195
|
+
# their presence/absence can be asserted by your tests.
|
196
|
+
#
|
197
|
+
# This is similar to ActionMailer's :test delivery_method and its
|
198
|
+
# ActionMailer::Base.deliveries array.
|
199
|
+
#
|
200
|
+
# Example:
|
201
|
+
#
|
202
|
+
# require 'sidekiq/testing'
|
203
|
+
#
|
204
|
+
# assert_equal 0, HardWorker.jobs.size
|
205
|
+
# HardWorker.perform_async(:something)
|
206
|
+
# assert_equal 1, HardWorker.jobs.size
|
207
|
+
# assert_equal :something, HardWorker.jobs[0]['args'][0]
|
208
|
+
#
|
209
|
+
# assert_equal 0, Sidekiq::Extensions::DelayedMailer.jobs.size
|
210
|
+
# MyMailer.delay.send_welcome_email('foo@example.com')
|
211
|
+
# assert_equal 1, Sidekiq::Extensions::DelayedMailer.jobs.size
|
212
|
+
#
|
213
|
+
# You can also clear and drain all workers' jobs:
|
214
|
+
#
|
215
|
+
# assert_equal 0, Sidekiq::Extensions::DelayedMailer.jobs.size
|
216
|
+
# assert_equal 0, Sidekiq::Extensions::DelayedModel.jobs.size
|
217
|
+
#
|
218
|
+
# MyMailer.delay.send_welcome_email('foo@example.com')
|
219
|
+
# MyModel.delay.do_something_hard
|
220
|
+
#
|
221
|
+
# assert_equal 1, Sidekiq::Extensions::DelayedMailer.jobs.size
|
222
|
+
# assert_equal 1, Sidekiq::Extensions::DelayedModel.jobs.size
|
223
|
+
#
|
224
|
+
# Sidekiq::Worker.clear_all # or .drain_all
|
225
|
+
#
|
226
|
+
# assert_equal 0, Sidekiq::Extensions::DelayedMailer.jobs.size
|
227
|
+
# assert_equal 0, Sidekiq::Extensions::DelayedModel.jobs.size
|
228
|
+
#
|
229
|
+
# This can be useful to make sure jobs don't linger between tests:
|
230
|
+
#
|
231
|
+
# RSpec.configure do |config|
|
232
|
+
# config.before(:each) do
|
233
|
+
# Sidekiq::Worker.clear_all
|
234
|
+
# end
|
235
|
+
# end
|
236
|
+
#
|
237
|
+
# or for acceptance testing, i.e. with cucumber:
|
238
|
+
#
|
239
|
+
# AfterStep do
|
240
|
+
# Sidekiq::Worker.drain_all
|
241
|
+
# end
|
242
|
+
#
|
243
|
+
# When I sign up as "foo@example.com"
|
244
|
+
# Then I should receive a welcome email to "foo@example.com"
|
245
|
+
#
|
246
|
+
module ClassMethods
|
247
|
+
|
248
|
+
# Queue for this worker
|
249
|
+
def queue
|
250
|
+
self.sidekiq_options["queue"]
|
251
|
+
end
|
252
|
+
|
253
|
+
# Jobs queued for this worker
|
254
|
+
def jobs
|
255
|
+
Queues.jobs_by_worker[self.to_s]
|
256
|
+
end
|
257
|
+
|
258
|
+
# Clear all jobs for this worker
|
259
|
+
def clear
|
260
|
+
Queues.clear_for(queue, self.to_s)
|
261
|
+
end
|
262
|
+
|
263
|
+
# Drain and run all jobs for this worker
|
264
|
+
def drain
|
265
|
+
while jobs.any?
|
266
|
+
next_job = jobs.first
|
267
|
+
Queues.delete_for(next_job["jid"], queue, self.to_s)
|
268
|
+
process_job(next_job)
|
269
|
+
end
|
270
|
+
end
|
271
|
+
|
272
|
+
# Pop out a single job and perform it
|
273
|
+
def perform_one
|
274
|
+
raise(EmptyQueueError, "perform_one called with empty job queue") if jobs.empty?
|
275
|
+
next_job = jobs.first
|
276
|
+
Queues.delete_for(next_job["jid"], queue, self.to_s)
|
277
|
+
process_job(next_job)
|
278
|
+
end
|
279
|
+
|
280
|
+
def process_job(job)
|
281
|
+
worker = new
|
282
|
+
worker.jid = job['jid']
|
283
|
+
worker.bid = job['bid'] if worker.respond_to?(:bid=)
|
284
|
+
Sidekiq::Testing.server_middleware.invoke(worker, job, job['queue']) do
|
285
|
+
execute_job(worker, job['args'])
|
286
|
+
end
|
287
|
+
end
|
288
|
+
|
289
|
+
def execute_job(worker, args)
|
290
|
+
worker.perform(*args)
|
291
|
+
end
|
292
|
+
end
|
293
|
+
|
294
|
+
class << self
|
295
|
+
def jobs # :nodoc:
|
296
|
+
Queues.jobs_by_queue.values.flatten
|
297
|
+
end
|
298
|
+
|
299
|
+
# Clear all queued jobs across all workers
|
300
|
+
def clear_all
|
301
|
+
Queues.clear_all
|
302
|
+
end
|
303
|
+
|
304
|
+
# Drain all queued jobs across all workers
|
305
|
+
def drain_all
|
306
|
+
while jobs.any?
|
307
|
+
worker_classes = jobs.map { |job| job["class"] }.uniq
|
308
|
+
|
309
|
+
worker_classes.each do |worker_class|
|
310
|
+
worker_class.constantize.drain
|
311
|
+
end
|
312
|
+
end
|
313
|
+
end
|
314
|
+
end
|
315
|
+
end
|
316
|
+
end
|