sidekiq_cleaner 5.3.6
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +7 -0
- data/.circleci/config.yml +61 -0
- data/.github/contributing.md +32 -0
- data/.github/issue_template.md +11 -0
- data/.gitignore +15 -0
- data/.travis.yml +11 -0
- data/3.0-Upgrade.md +70 -0
- data/4.0-Upgrade.md +53 -0
- data/5.0-Upgrade.md +56 -0
- data/COMM-LICENSE +97 -0
- data/Changes.md +1536 -0
- data/Ent-Changes.md +238 -0
- data/Gemfile +23 -0
- data/LICENSE +9 -0
- data/Pro-2.0-Upgrade.md +138 -0
- data/Pro-3.0-Upgrade.md +44 -0
- data/Pro-4.0-Upgrade.md +35 -0
- data/Pro-Changes.md +759 -0
- data/README.md +55 -0
- data/Rakefile +9 -0
- data/bin/sidekiq +18 -0
- data/bin/sidekiqctl +20 -0
- data/bin/sidekiqload +149 -0
- data/cleaner/assets/images/favicon.ico +0 -0
- data/cleaner/assets/images/logo.png +0 -0
- data/cleaner/assets/images/status.png +0 -0
- data/cleaner/assets/javascripts/application.js +172 -0
- data/cleaner/assets/javascripts/dashboard.js +315 -0
- data/cleaner/assets/stylesheets/application-rtl.css +246 -0
- data/cleaner/assets/stylesheets/application.css +1144 -0
- data/cleaner/assets/stylesheets/bootstrap-rtl.min.css +9 -0
- data/cleaner/assets/stylesheets/bootstrap.css +5 -0
- data/cleaner/locales/ar.yml +81 -0
- data/cleaner/locales/cs.yml +78 -0
- data/cleaner/locales/da.yml +68 -0
- data/cleaner/locales/de.yml +69 -0
- data/cleaner/locales/el.yml +68 -0
- data/cleaner/locales/en.yml +81 -0
- data/cleaner/locales/es.yml +70 -0
- data/cleaner/locales/fa.yml +80 -0
- data/cleaner/locales/fr.yml +78 -0
- data/cleaner/locales/he.yml +79 -0
- data/cleaner/locales/hi.yml +75 -0
- data/cleaner/locales/it.yml +69 -0
- data/cleaner/locales/ja.yml +80 -0
- data/cleaner/locales/ko.yml +68 -0
- data/cleaner/locales/nb.yml +77 -0
- data/cleaner/locales/nl.yml +68 -0
- data/cleaner/locales/pl.yml +59 -0
- data/cleaner/locales/pt-br.yml +68 -0
- data/cleaner/locales/pt.yml +67 -0
- data/cleaner/locales/ru.yml +78 -0
- data/cleaner/locales/sv.yml +68 -0
- data/cleaner/locales/ta.yml +75 -0
- data/cleaner/locales/uk.yml +76 -0
- data/cleaner/locales/ur.yml +80 -0
- data/cleaner/locales/zh-cn.yml +68 -0
- data/cleaner/locales/zh-tw.yml +68 -0
- data/cleaner/views/_footer.erb +20 -0
- data/cleaner/views/_job_info.erb +88 -0
- data/cleaner/views/_nav.erb +52 -0
- data/cleaner/views/_paging.erb +23 -0
- data/cleaner/views/_poll_link.erb +7 -0
- data/cleaner/views/_status.erb +4 -0
- data/cleaner/views/_summary.erb +40 -0
- data/cleaner/views/busy.erb +98 -0
- data/cleaner/views/dashboard.erb +75 -0
- data/cleaner/views/dead.erb +34 -0
- data/cleaner/views/errors.erb +84 -0
- data/cleaner/views/layout.erb +40 -0
- data/cleaner/views/morgue.erb +75 -0
- data/cleaner/views/queue.erb +46 -0
- data/cleaner/views/queues.erb +30 -0
- data/cleaner/views/retries.erb +80 -0
- data/cleaner/views/retry.erb +34 -0
- data/cleaner/views/scheduled.erb +54 -0
- data/cleaner/views/scheduled_job_info.erb +8 -0
- data/cleaner-stats.png +0 -0
- data/cleaner.png +0 -0
- data/code_of_conduct.md +50 -0
- data/lib/generators/sidekiq/templates/worker.rb.erb +9 -0
- data/lib/generators/sidekiq/templates/worker_spec.rb.erb +6 -0
- data/lib/generators/sidekiq/templates/worker_test.rb.erb +8 -0
- data/lib/generators/sidekiq/worker_generator.rb +49 -0
- data/lib/sidekiq/api.rb +940 -0
- data/lib/sidekiq/cleaner/action.rb +89 -0
- data/lib/sidekiq/cleaner/application.rb +385 -0
- data/lib/sidekiq/cleaner/helpers.rb +325 -0
- data/lib/sidekiq/cleaner/router.rb +100 -0
- data/lib/sidekiq/cleaner.rb +214 -0
- data/lib/sidekiq/cli.rb +445 -0
- data/lib/sidekiq/client.rb +243 -0
- data/lib/sidekiq/core_ext.rb +1 -0
- data/lib/sidekiq/ctl.rb +221 -0
- data/lib/sidekiq/delay.rb +42 -0
- data/lib/sidekiq/exception_handler.rb +29 -0
- data/lib/sidekiq/extensions/action_mailer.rb +57 -0
- data/lib/sidekiq/extensions/active_record.rb +40 -0
- data/lib/sidekiq/extensions/class_methods.rb +40 -0
- data/lib/sidekiq/extensions/generic_proxy.rb +31 -0
- data/lib/sidekiq/fetch.rb +81 -0
- data/lib/sidekiq/job_logger.rb +25 -0
- data/lib/sidekiq/job_retry.rb +262 -0
- data/lib/sidekiq/launcher.rb +173 -0
- data/lib/sidekiq/logging.rb +122 -0
- data/lib/sidekiq/manager.rb +137 -0
- data/lib/sidekiq/middleware/chain.rb +150 -0
- data/lib/sidekiq/middleware/i18n.rb +42 -0
- data/lib/sidekiq/middleware/server/active_record.rb +23 -0
- data/lib/sidekiq/paginator.rb +43 -0
- data/lib/sidekiq/processor.rb +279 -0
- data/lib/sidekiq/rails.rb +58 -0
- data/lib/sidekiq/redis_connection.rb +144 -0
- data/lib/sidekiq/scheduled.rb +174 -0
- data/lib/sidekiq/testing/inline.rb +29 -0
- data/lib/sidekiq/testing.rb +333 -0
- data/lib/sidekiq/util.rb +66 -0
- data/lib/sidekiq/version.rb +4 -0
- data/lib/sidekiq/worker.rb +220 -0
- data/lib/sidekiq.rb +237 -0
- data/sidekiq_cleaner.gemspec +21 -0
- metadata +235 -0
@@ -0,0 +1,150 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
module Sidekiq
|
3
|
+
# Middleware is code configured to run before/after
|
4
|
+
# a message is processed. It is patterned after Rack
|
5
|
+
# middleware. Middleware exists for the client side
|
6
|
+
# (pushing jobs onto the queue) as well as the server
|
7
|
+
# side (when jobs are actually processed).
|
8
|
+
#
|
9
|
+
# To add middleware for the client:
|
10
|
+
#
|
11
|
+
# Sidekiq.configure_client do |config|
|
12
|
+
# config.client_middleware do |chain|
|
13
|
+
# chain.add MyClientHook
|
14
|
+
# end
|
15
|
+
# end
|
16
|
+
#
|
17
|
+
# To modify middleware for the server, just call
|
18
|
+
# with another block:
|
19
|
+
#
|
20
|
+
# Sidekiq.configure_server do |config|
|
21
|
+
# config.server_middleware do |chain|
|
22
|
+
# chain.add MyServerHook
|
23
|
+
# chain.remove ActiveRecord
|
24
|
+
# end
|
25
|
+
# end
|
26
|
+
#
|
27
|
+
# To insert immediately preceding another entry:
|
28
|
+
#
|
29
|
+
# Sidekiq.configure_client do |config|
|
30
|
+
# config.client_middleware do |chain|
|
31
|
+
# chain.insert_before ActiveRecord, MyClientHook
|
32
|
+
# end
|
33
|
+
# end
|
34
|
+
#
|
35
|
+
# To insert immediately after another entry:
|
36
|
+
#
|
37
|
+
# Sidekiq.configure_client do |config|
|
38
|
+
# config.client_middleware do |chain|
|
39
|
+
# chain.insert_after ActiveRecord, MyClientHook
|
40
|
+
# end
|
41
|
+
# end
|
42
|
+
#
|
43
|
+
# This is an example of a minimal server middleware:
|
44
|
+
#
|
45
|
+
# class MyServerHook
|
46
|
+
# def call(worker_instance, msg, queue)
|
47
|
+
# puts "Before work"
|
48
|
+
# yield
|
49
|
+
# puts "After work"
|
50
|
+
# end
|
51
|
+
# end
|
52
|
+
#
|
53
|
+
# This is an example of a minimal client middleware, note
|
54
|
+
# the method must return the result or the job will not push
|
55
|
+
# to Redis:
|
56
|
+
#
|
57
|
+
# class MyClientHook
|
58
|
+
# def call(worker_class, msg, queue, redis_pool)
|
59
|
+
# puts "Before push"
|
60
|
+
# result = yield
|
61
|
+
# puts "After push"
|
62
|
+
# result
|
63
|
+
# end
|
64
|
+
# end
|
65
|
+
#
|
66
|
+
module Middleware
|
67
|
+
class Chain
|
68
|
+
include Enumerable
|
69
|
+
attr_reader :entries
|
70
|
+
|
71
|
+
def initialize_copy(copy)
|
72
|
+
copy.instance_variable_set(:@entries, entries.dup)
|
73
|
+
end
|
74
|
+
|
75
|
+
def each(&block)
|
76
|
+
entries.each(&block)
|
77
|
+
end
|
78
|
+
|
79
|
+
def initialize
|
80
|
+
@entries = []
|
81
|
+
yield self if block_given?
|
82
|
+
end
|
83
|
+
|
84
|
+
def remove(klass)
|
85
|
+
entries.delete_if { |entry| entry.klass == klass }
|
86
|
+
end
|
87
|
+
|
88
|
+
def add(klass, *args)
|
89
|
+
remove(klass) if exists?(klass)
|
90
|
+
entries << Entry.new(klass, *args)
|
91
|
+
end
|
92
|
+
|
93
|
+
def prepend(klass, *args)
|
94
|
+
remove(klass) if exists?(klass)
|
95
|
+
entries.insert(0, Entry.new(klass, *args))
|
96
|
+
end
|
97
|
+
|
98
|
+
def insert_before(oldklass, newklass, *args)
|
99
|
+
i = entries.index { |entry| entry.klass == newklass }
|
100
|
+
new_entry = i.nil? ? Entry.new(newklass, *args) : entries.delete_at(i)
|
101
|
+
i = entries.index { |entry| entry.klass == oldklass } || 0
|
102
|
+
entries.insert(i, new_entry)
|
103
|
+
end
|
104
|
+
|
105
|
+
def insert_after(oldklass, newklass, *args)
|
106
|
+
i = entries.index { |entry| entry.klass == newklass }
|
107
|
+
new_entry = i.nil? ? Entry.new(newklass, *args) : entries.delete_at(i)
|
108
|
+
i = entries.index { |entry| entry.klass == oldklass } || entries.count - 1
|
109
|
+
entries.insert(i+1, new_entry)
|
110
|
+
end
|
111
|
+
|
112
|
+
def exists?(klass)
|
113
|
+
any? { |entry| entry.klass == klass }
|
114
|
+
end
|
115
|
+
|
116
|
+
def retrieve
|
117
|
+
map(&:make_new)
|
118
|
+
end
|
119
|
+
|
120
|
+
def clear
|
121
|
+
entries.clear
|
122
|
+
end
|
123
|
+
|
124
|
+
def invoke(*args)
|
125
|
+
chain = retrieve.dup
|
126
|
+
traverse_chain = lambda do
|
127
|
+
if chain.empty?
|
128
|
+
yield
|
129
|
+
else
|
130
|
+
chain.shift.call(*args, &traverse_chain)
|
131
|
+
end
|
132
|
+
end
|
133
|
+
traverse_chain.call
|
134
|
+
end
|
135
|
+
end
|
136
|
+
|
137
|
+
class Entry
|
138
|
+
attr_reader :klass
|
139
|
+
|
140
|
+
def initialize(klass, *args)
|
141
|
+
@klass = klass
|
142
|
+
@args = args
|
143
|
+
end
|
144
|
+
|
145
|
+
def make_new
|
146
|
+
@klass.new(*@args)
|
147
|
+
end
|
148
|
+
end
|
149
|
+
end
|
150
|
+
end
|
@@ -0,0 +1,42 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
#
|
3
|
+
# Simple middleware to save the current locale and restore it when the job executes.
|
4
|
+
# Use it by requiring it in your initializer:
|
5
|
+
#
|
6
|
+
# require 'sidekiq/middleware/i18n'
|
7
|
+
#
|
8
|
+
module Sidekiq::Middleware::I18n
|
9
|
+
# Get the current locale and store it in the message
|
10
|
+
# to be sent to Sidekiq.
|
11
|
+
class Client
|
12
|
+
def call(worker_class, msg, queue, redis_pool)
|
13
|
+
msg['locale'] ||= I18n.locale
|
14
|
+
yield
|
15
|
+
end
|
16
|
+
end
|
17
|
+
|
18
|
+
# Pull the msg locale out and set the current thread to use it.
|
19
|
+
class Server
|
20
|
+
def call(worker, msg, queue)
|
21
|
+
I18n.locale = msg['locale'] || I18n.default_locale
|
22
|
+
yield
|
23
|
+
ensure
|
24
|
+
I18n.locale = I18n.default_locale
|
25
|
+
end
|
26
|
+
end
|
27
|
+
end
|
28
|
+
|
29
|
+
Sidekiq.configure_client do |config|
|
30
|
+
config.client_middleware do |chain|
|
31
|
+
chain.add Sidekiq::Middleware::I18n::Client
|
32
|
+
end
|
33
|
+
end
|
34
|
+
|
35
|
+
Sidekiq.configure_server do |config|
|
36
|
+
config.client_middleware do |chain|
|
37
|
+
chain.add Sidekiq::Middleware::I18n::Client
|
38
|
+
end
|
39
|
+
config.server_middleware do |chain|
|
40
|
+
chain.add Sidekiq::Middleware::I18n::Server
|
41
|
+
end
|
42
|
+
end
|
@@ -0,0 +1,23 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
module Sidekiq
|
3
|
+
module Middleware
|
4
|
+
module Server
|
5
|
+
class ActiveRecord
|
6
|
+
|
7
|
+
def initialize
|
8
|
+
# With Rails 5+ we must use the Reloader **always**.
|
9
|
+
# The reloader handles code loading and db connection management.
|
10
|
+
if defined?(::Rails) && ::Rails::VERSION::MAJOR >= 5
|
11
|
+
raise ArgumentError, "Rails 5 no longer needs or uses the ActiveRecord middleware."
|
12
|
+
end
|
13
|
+
end
|
14
|
+
|
15
|
+
def call(*args)
|
16
|
+
yield
|
17
|
+
ensure
|
18
|
+
::ActiveRecord::Base.clear_active_connections!
|
19
|
+
end
|
20
|
+
end
|
21
|
+
end
|
22
|
+
end
|
23
|
+
end
|
@@ -0,0 +1,43 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
module Sidekiq
|
3
|
+
module Paginator
|
4
|
+
|
5
|
+
def page(key, pageidx=1, page_size=25, opts=nil)
|
6
|
+
current_page = pageidx.to_i < 1 ? 1 : pageidx.to_i
|
7
|
+
pageidx = current_page - 1
|
8
|
+
total_size = 0
|
9
|
+
items = []
|
10
|
+
starting = pageidx * page_size
|
11
|
+
ending = starting + page_size - 1
|
12
|
+
|
13
|
+
Sidekiq.redis do |conn|
|
14
|
+
type = conn.type(key)
|
15
|
+
|
16
|
+
case type
|
17
|
+
when 'zset'
|
18
|
+
rev = opts && opts[:reverse]
|
19
|
+
total_size, items = conn.multi do
|
20
|
+
conn.zcard(key)
|
21
|
+
if rev
|
22
|
+
conn.zrevrange(key, starting, ending, :with_scores => true)
|
23
|
+
else
|
24
|
+
conn.zrange(key, starting, ending, :with_scores => true)
|
25
|
+
end
|
26
|
+
end
|
27
|
+
[current_page, total_size, items]
|
28
|
+
when 'list'
|
29
|
+
total_size, items = conn.multi do
|
30
|
+
conn.llen(key)
|
31
|
+
conn.lrange(key, starting, ending)
|
32
|
+
end
|
33
|
+
[current_page, total_size, items]
|
34
|
+
when 'none'
|
35
|
+
[1, 0, []]
|
36
|
+
else
|
37
|
+
raise "can't page a #{type}"
|
38
|
+
end
|
39
|
+
end
|
40
|
+
end
|
41
|
+
|
42
|
+
end
|
43
|
+
end
|
@@ -0,0 +1,279 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
require 'sidekiq/util'
|
3
|
+
require 'sidekiq/fetch'
|
4
|
+
require 'sidekiq/job_logger'
|
5
|
+
require 'sidekiq/job_retry'
|
6
|
+
require 'thread'
|
7
|
+
|
8
|
+
module Sidekiq
|
9
|
+
##
|
10
|
+
# The Processor is a standalone thread which:
|
11
|
+
#
|
12
|
+
# 1. fetches a job from Redis
|
13
|
+
# 2. executes the job
|
14
|
+
# a. instantiate the Worker
|
15
|
+
# b. run the middleware chain
|
16
|
+
# c. call #perform
|
17
|
+
#
|
18
|
+
# A Processor can exit due to shutdown (processor_stopped)
|
19
|
+
# or due to an error during job execution (processor_died)
|
20
|
+
#
|
21
|
+
# If an error occurs in the job execution, the
|
22
|
+
# Processor calls the Manager to create a new one
|
23
|
+
# to replace itself and exits.
|
24
|
+
#
|
25
|
+
class Processor
|
26
|
+
|
27
|
+
include Util
|
28
|
+
|
29
|
+
attr_reader :thread
|
30
|
+
attr_reader :job
|
31
|
+
|
32
|
+
def initialize(mgr)
|
33
|
+
@mgr = mgr
|
34
|
+
@down = false
|
35
|
+
@done = false
|
36
|
+
@job = nil
|
37
|
+
@thread = nil
|
38
|
+
@strategy = (mgr.options[:fetch] || Sidekiq::BasicFetch).new(mgr.options)
|
39
|
+
@reloader = Sidekiq.options[:reloader]
|
40
|
+
@logging = (mgr.options[:job_logger] || Sidekiq::JobLogger).new
|
41
|
+
@retrier = Sidekiq::JobRetry.new
|
42
|
+
end
|
43
|
+
|
44
|
+
def terminate(wait=false)
|
45
|
+
@done = true
|
46
|
+
return if !@thread
|
47
|
+
@thread.value if wait
|
48
|
+
end
|
49
|
+
|
50
|
+
def kill(wait=false)
|
51
|
+
@done = true
|
52
|
+
return if !@thread
|
53
|
+
# unlike the other actors, terminate does not wait
|
54
|
+
# for the thread to finish because we don't know how
|
55
|
+
# long the job will take to finish. Instead we
|
56
|
+
# provide a `kill` method to call after the shutdown
|
57
|
+
# timeout passes.
|
58
|
+
@thread.raise ::Sidekiq::Shutdown
|
59
|
+
@thread.value if wait
|
60
|
+
end
|
61
|
+
|
62
|
+
def start
|
63
|
+
@thread ||= safe_thread("processor", &method(:run))
|
64
|
+
end
|
65
|
+
|
66
|
+
private unless $TESTING
|
67
|
+
|
68
|
+
def run
|
69
|
+
begin
|
70
|
+
while !@done
|
71
|
+
process_one
|
72
|
+
end
|
73
|
+
@mgr.processor_stopped(self)
|
74
|
+
rescue Sidekiq::Shutdown
|
75
|
+
@mgr.processor_stopped(self)
|
76
|
+
rescue Exception => ex
|
77
|
+
@mgr.processor_died(self, ex)
|
78
|
+
end
|
79
|
+
end
|
80
|
+
|
81
|
+
def process_one
|
82
|
+
@job = fetch
|
83
|
+
process(@job) if @job
|
84
|
+
@job = nil
|
85
|
+
end
|
86
|
+
|
87
|
+
def get_one
|
88
|
+
begin
|
89
|
+
work = @strategy.retrieve_work
|
90
|
+
(logger.info { "Redis is online, #{::Process.clock_gettime(::Process::CLOCK_MONOTONIC) - @down} sec downtime" }; @down = nil) if @down
|
91
|
+
work
|
92
|
+
rescue Sidekiq::Shutdown
|
93
|
+
rescue => ex
|
94
|
+
handle_fetch_exception(ex)
|
95
|
+
end
|
96
|
+
end
|
97
|
+
|
98
|
+
def fetch
|
99
|
+
j = get_one
|
100
|
+
if j && @done
|
101
|
+
j.requeue
|
102
|
+
nil
|
103
|
+
else
|
104
|
+
j
|
105
|
+
end
|
106
|
+
end
|
107
|
+
|
108
|
+
def handle_fetch_exception(ex)
|
109
|
+
if !@down
|
110
|
+
@down = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
|
111
|
+
logger.error("Error fetching job: #{ex}")
|
112
|
+
handle_exception(ex)
|
113
|
+
end
|
114
|
+
sleep(1)
|
115
|
+
nil
|
116
|
+
end
|
117
|
+
|
118
|
+
def dispatch(job_hash, queue)
|
119
|
+
# since middleware can mutate the job hash
|
120
|
+
# we clone here so we report the original
|
121
|
+
# job structure to the Web UI
|
122
|
+
pristine = cloned(job_hash)
|
123
|
+
|
124
|
+
Sidekiq::Logging.with_job_hash_context(job_hash) do
|
125
|
+
@retrier.global(pristine, queue) do
|
126
|
+
@logging.call(job_hash, queue) do
|
127
|
+
stats(pristine, queue) do
|
128
|
+
# Rails 5 requires a Reloader to wrap code execution. In order to
|
129
|
+
# constantize the worker and instantiate an instance, we have to call
|
130
|
+
# the Reloader. It handles code loading, db connection management, etc.
|
131
|
+
# Effectively this block denotes a "unit of work" to Rails.
|
132
|
+
@reloader.call do
|
133
|
+
klass = constantize(job_hash['class'])
|
134
|
+
worker = klass.new
|
135
|
+
worker.jid = job_hash['jid']
|
136
|
+
@retrier.local(worker, pristine, queue) do
|
137
|
+
yield worker
|
138
|
+
end
|
139
|
+
end
|
140
|
+
end
|
141
|
+
end
|
142
|
+
end
|
143
|
+
end
|
144
|
+
end
|
145
|
+
|
146
|
+
def process(work)
|
147
|
+
jobstr = work.job
|
148
|
+
queue = work.queue_name
|
149
|
+
|
150
|
+
# Treat malformed JSON as a special case: job goes straight to the morgue.
|
151
|
+
job_hash = nil
|
152
|
+
begin
|
153
|
+
job_hash = Sidekiq.load_json(jobstr)
|
154
|
+
rescue => ex
|
155
|
+
handle_exception(ex, { :context => "Invalid JSON for job", :jobstr => jobstr })
|
156
|
+
# we can't notify because the job isn't a valid hash payload.
|
157
|
+
DeadSet.new.kill(jobstr, notify_failure: false)
|
158
|
+
return work.acknowledge
|
159
|
+
end
|
160
|
+
|
161
|
+
ack = true
|
162
|
+
begin
|
163
|
+
dispatch(job_hash, queue) do |worker|
|
164
|
+
Sidekiq.server_middleware.invoke(worker, job_hash, queue) do
|
165
|
+
execute_job(worker, cloned(job_hash['args']))
|
166
|
+
end
|
167
|
+
end
|
168
|
+
rescue Sidekiq::Shutdown
|
169
|
+
# Had to force kill this job because it didn't finish
|
170
|
+
# within the timeout. Don't acknowledge the work since
|
171
|
+
# we didn't properly finish it.
|
172
|
+
ack = false
|
173
|
+
rescue Sidekiq::JobRetry::Handled => h
|
174
|
+
# this is the common case: job raised error and Sidekiq::JobRetry::Handled
|
175
|
+
# signals that we created a retry successfully. We can acknowlege the job.
|
176
|
+
e = h.cause ? h.cause : h
|
177
|
+
handle_exception(e, { :context => "Job raised exception", :job => job_hash, :jobstr => jobstr })
|
178
|
+
raise e
|
179
|
+
rescue Exception => ex
|
180
|
+
# Unexpected error! This is very bad and indicates an exception that got past
|
181
|
+
# the retry subsystem (e.g. network partition). We won't acknowledge the job
|
182
|
+
# so it can be rescued when using Sidekiq Pro.
|
183
|
+
ack = false
|
184
|
+
handle_exception(ex, { :context => "Internal exception!", :job => job_hash, :jobstr => jobstr })
|
185
|
+
raise e
|
186
|
+
ensure
|
187
|
+
work.acknowledge if ack
|
188
|
+
end
|
189
|
+
end
|
190
|
+
|
191
|
+
def execute_job(worker, cloned_args)
|
192
|
+
worker.perform(*cloned_args)
|
193
|
+
end
|
194
|
+
|
195
|
+
# Ruby doesn't provide atomic counters out of the box so we'll
|
196
|
+
# implement something simple ourselves.
|
197
|
+
# https://bugs.ruby-lang.org/issues/14706
|
198
|
+
class Counter
|
199
|
+
def initialize
|
200
|
+
@value = 0
|
201
|
+
@lock = Mutex.new
|
202
|
+
end
|
203
|
+
|
204
|
+
def incr(amount=1)
|
205
|
+
@lock.synchronize { @value = @value + amount }
|
206
|
+
end
|
207
|
+
|
208
|
+
def reset
|
209
|
+
@lock.synchronize { val = @value; @value = 0; val }
|
210
|
+
end
|
211
|
+
end
|
212
|
+
|
213
|
+
# jruby's Hash implementation is not threadsafe, so we wrap it in a mutex here
|
214
|
+
class SharedWorkerState
|
215
|
+
def initialize
|
216
|
+
@worker_state = {}
|
217
|
+
@lock = Mutex.new
|
218
|
+
end
|
219
|
+
|
220
|
+
def set(tid, hash)
|
221
|
+
@lock.synchronize { @worker_state[tid] = hash }
|
222
|
+
end
|
223
|
+
|
224
|
+
def delete(tid)
|
225
|
+
@lock.synchronize { @worker_state.delete(tid) }
|
226
|
+
end
|
227
|
+
|
228
|
+
def dup
|
229
|
+
@lock.synchronize { @worker_state.dup }
|
230
|
+
end
|
231
|
+
|
232
|
+
def size
|
233
|
+
@lock.synchronize { @worker_state.size }
|
234
|
+
end
|
235
|
+
|
236
|
+
def clear
|
237
|
+
@lock.synchronize { @worker_state.clear }
|
238
|
+
end
|
239
|
+
end
|
240
|
+
|
241
|
+
PROCESSED = Counter.new
|
242
|
+
FAILURE = Counter.new
|
243
|
+
WORKER_STATE = SharedWorkerState.new
|
244
|
+
|
245
|
+
def stats(job_hash, queue)
|
246
|
+
tid = Sidekiq::Logging.tid
|
247
|
+
WORKER_STATE.set(tid, {:queue => queue, :payload => job_hash, :run_at => Time.now.to_i })
|
248
|
+
|
249
|
+
begin
|
250
|
+
yield
|
251
|
+
rescue Exception
|
252
|
+
FAILURE.incr
|
253
|
+
raise
|
254
|
+
ensure
|
255
|
+
WORKER_STATE.delete(tid)
|
256
|
+
PROCESSED.incr
|
257
|
+
end
|
258
|
+
end
|
259
|
+
|
260
|
+
# Deep clone the arguments passed to the worker so that if
|
261
|
+
# the job fails, what is pushed back onto Redis hasn't
|
262
|
+
# been mutated by the worker.
|
263
|
+
def cloned(thing)
|
264
|
+
Marshal.load(Marshal.dump(thing))
|
265
|
+
end
|
266
|
+
|
267
|
+
def constantize(str)
|
268
|
+
names = str.split('::')
|
269
|
+
names.shift if names.empty? || names.first.empty?
|
270
|
+
|
271
|
+
names.inject(Object) do |constant, name|
|
272
|
+
# the false flag limits search for name to under the constant namespace
|
273
|
+
# which mimics Rails' behaviour
|
274
|
+
constant.const_defined?(name, false) ? constant.const_get(name, false) : constant.const_missing(name)
|
275
|
+
end
|
276
|
+
end
|
277
|
+
|
278
|
+
end
|
279
|
+
end
|
@@ -0,0 +1,58 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Sidekiq
|
4
|
+
class Rails < ::Rails::Engine
|
5
|
+
# We need to setup this up before any application configuration which might
|
6
|
+
# change Sidekiq middleware.
|
7
|
+
#
|
8
|
+
# This hook happens after `Rails::Application` is inherited within
|
9
|
+
# config/application.rb and before config is touched, usually within the
|
10
|
+
# class block. Definitely before config/environments/*.rb and
|
11
|
+
# config/initializers/*.rb.
|
12
|
+
config.before_configuration do
|
13
|
+
if ::Rails::VERSION::MAJOR < 5 && defined?(::ActiveRecord)
|
14
|
+
Sidekiq.server_middleware do |chain|
|
15
|
+
require 'sidekiq/middleware/server/active_record'
|
16
|
+
chain.add Sidekiq::Middleware::Server::ActiveRecord
|
17
|
+
end
|
18
|
+
end
|
19
|
+
end
|
20
|
+
|
21
|
+
config.after_initialize do
|
22
|
+
# This hook happens after all initializers are run, just before returning
|
23
|
+
# from config/environment.rb back to sidekiq/cli.rb.
|
24
|
+
# We have to add the reloader after initialize to see if cache_classes has
|
25
|
+
# been turned on.
|
26
|
+
#
|
27
|
+
# None of this matters on the client-side, only within the Sidekiq process itself.
|
28
|
+
#
|
29
|
+
Sidekiq.configure_server do |_|
|
30
|
+
if ::Rails::VERSION::MAJOR >= 5
|
31
|
+
Sidekiq.options[:reloader] = Sidekiq::Rails::Reloader.new
|
32
|
+
end
|
33
|
+
end
|
34
|
+
end
|
35
|
+
|
36
|
+
class Reloader
|
37
|
+
def initialize(app = ::Rails.application)
|
38
|
+
@app = app
|
39
|
+
end
|
40
|
+
|
41
|
+
def call
|
42
|
+
@app.reloader.wrap do
|
43
|
+
yield
|
44
|
+
end
|
45
|
+
end
|
46
|
+
|
47
|
+
def inspect
|
48
|
+
"#<Sidekiq::Rails::Reloader @app=#{@app.class.name}>"
|
49
|
+
end
|
50
|
+
end
|
51
|
+
end if defined?(::Rails)
|
52
|
+
end
|
53
|
+
|
54
|
+
if defined?(::Rails) && ::Rails::VERSION::MAJOR < 4
|
55
|
+
$stderr.puts("**************************************************")
|
56
|
+
$stderr.puts("⛔️ WARNING: Sidekiq server is no longer supported by Rails 3.2 - please ensure your server/workers are updated")
|
57
|
+
$stderr.puts("**************************************************")
|
58
|
+
end
|