sr-sidekiq 4.1.6
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +7 -0
- data/.gitignore +12 -0
- data/3.0-Upgrade.md +70 -0
- data/4.0-Upgrade.md +50 -0
- data/COMM-LICENSE (sidekiq) +95 -0
- data/Changes.md +1241 -0
- data/Ent-Changes.md +112 -0
- data/Gemfile +29 -0
- data/LICENSE (sidekiq) +9 -0
- data/LICENSE (sr-sidekiq) +5 -0
- data/Pro-2.0-Upgrade.md +138 -0
- data/Pro-3.0-Upgrade.md +44 -0
- data/Pro-Changes.md +539 -0
- data/README.md +8 -0
- data/Rakefile +9 -0
- data/bin/sidekiq +18 -0
- data/bin/sidekiqctl +99 -0
- data/bin/sidekiqload +167 -0
- data/code_of_conduct.md +50 -0
- data/lib/generators/sidekiq/templates/worker.rb.erb +9 -0
- data/lib/generators/sidekiq/templates/worker_spec.rb.erb +6 -0
- data/lib/generators/sidekiq/templates/worker_test.rb.erb +8 -0
- data/lib/generators/sidekiq/worker_generator.rb +49 -0
- data/lib/sidekiq.rb +237 -0
- data/lib/sidekiq/api.rb +844 -0
- data/lib/sidekiq/cli.rb +389 -0
- data/lib/sidekiq/client.rb +260 -0
- data/lib/sidekiq/core_ext.rb +106 -0
- data/lib/sidekiq/exception_handler.rb +31 -0
- data/lib/sidekiq/extensions/action_mailer.rb +57 -0
- data/lib/sidekiq/extensions/active_record.rb +40 -0
- data/lib/sidekiq/extensions/class_methods.rb +40 -0
- data/lib/sidekiq/extensions/generic_proxy.rb +25 -0
- data/lib/sidekiq/fetch.rb +81 -0
- data/lib/sidekiq/launcher.rb +160 -0
- data/lib/sidekiq/logging.rb +106 -0
- data/lib/sidekiq/manager.rb +137 -0
- data/lib/sidekiq/middleware/chain.rb +150 -0
- data/lib/sidekiq/middleware/i18n.rb +42 -0
- data/lib/sidekiq/middleware/server/active_record.rb +13 -0
- data/lib/sidekiq/middleware/server/logging.rb +40 -0
- data/lib/sidekiq/middleware/server/retry_jobs.rb +205 -0
- data/lib/sidekiq/paginator.rb +43 -0
- data/lib/sidekiq/processor.rb +186 -0
- data/lib/sidekiq/rails.rb +39 -0
- data/lib/sidekiq/redis_connection.rb +97 -0
- data/lib/sidekiq/scheduled.rb +146 -0
- data/lib/sidekiq/testing.rb +316 -0
- data/lib/sidekiq/testing/inline.rb +29 -0
- data/lib/sidekiq/util.rb +62 -0
- data/lib/sidekiq/version.rb +4 -0
- data/lib/sidekiq/web.rb +278 -0
- data/lib/sidekiq/web_helpers.rb +255 -0
- data/lib/sidekiq/worker.rb +121 -0
- data/sidekiq.gemspec +26 -0
- data/sr-sidekiq-4.1.3.gem +0 -0
- data/sr-sidekiq-4.1.4.gem +0 -0
- data/sr-sidekiq-4.1.5.gem +0 -0
- data/test/config.yml +9 -0
- data/test/env_based_config.yml +11 -0
- data/test/fake_env.rb +1 -0
- data/test/fixtures/en.yml +2 -0
- data/test/helper.rb +75 -0
- data/test/test_actors.rb +138 -0
- data/test/test_api.rb +528 -0
- data/test/test_cli.rb +406 -0
- data/test/test_client.rb +262 -0
- data/test/test_exception_handler.rb +56 -0
- data/test/test_extensions.rb +127 -0
- data/test/test_fetch.rb +50 -0
- data/test/test_launcher.rb +85 -0
- data/test/test_logging.rb +35 -0
- data/test/test_manager.rb +50 -0
- data/test/test_middleware.rb +158 -0
- data/test/test_processor.rb +201 -0
- data/test/test_rails.rb +22 -0
- data/test/test_redis_connection.rb +127 -0
- data/test/test_retry.rb +326 -0
- data/test/test_retry_exhausted.rb +149 -0
- data/test/test_scheduled.rb +115 -0
- data/test/test_scheduling.rb +50 -0
- data/test/test_sidekiq.rb +107 -0
- data/test/test_testing.rb +143 -0
- data/test/test_testing_fake.rb +357 -0
- data/test/test_testing_inline.rb +94 -0
- data/test/test_util.rb +13 -0
- data/test/test_web.rb +614 -0
- data/test/test_web_helpers.rb +54 -0
- data/web/assets/images/bootstrap/glyphicons-halflings-white.png +0 -0
- data/web/assets/images/bootstrap/glyphicons-halflings.png +0 -0
- data/web/assets/images/favicon.ico +0 -0
- data/web/assets/images/logo.png +0 -0
- data/web/assets/images/status-sd8051fd480.png +0 -0
- data/web/assets/images/status/active.png +0 -0
- data/web/assets/images/status/idle.png +0 -0
- data/web/assets/javascripts/application.js +88 -0
- data/web/assets/javascripts/dashboard.js +300 -0
- data/web/assets/javascripts/locales/README.md +27 -0
- data/web/assets/javascripts/locales/jquery.timeago.ar.js +96 -0
- data/web/assets/javascripts/locales/jquery.timeago.bg.js +18 -0
- data/web/assets/javascripts/locales/jquery.timeago.bs.js +49 -0
- data/web/assets/javascripts/locales/jquery.timeago.ca.js +18 -0
- data/web/assets/javascripts/locales/jquery.timeago.cs.js +18 -0
- data/web/assets/javascripts/locales/jquery.timeago.cy.js +20 -0
- data/web/assets/javascripts/locales/jquery.timeago.da.js +18 -0
- data/web/assets/javascripts/locales/jquery.timeago.de.js +18 -0
- data/web/assets/javascripts/locales/jquery.timeago.el.js +18 -0
- data/web/assets/javascripts/locales/jquery.timeago.en-short.js +20 -0
- data/web/assets/javascripts/locales/jquery.timeago.en.js +20 -0
- data/web/assets/javascripts/locales/jquery.timeago.es.js +18 -0
- data/web/assets/javascripts/locales/jquery.timeago.et.js +18 -0
- data/web/assets/javascripts/locales/jquery.timeago.fa.js +22 -0
- data/web/assets/javascripts/locales/jquery.timeago.fi.js +28 -0
- data/web/assets/javascripts/locales/jquery.timeago.fr-short.js +16 -0
- data/web/assets/javascripts/locales/jquery.timeago.fr.js +17 -0
- data/web/assets/javascripts/locales/jquery.timeago.he.js +18 -0
- data/web/assets/javascripts/locales/jquery.timeago.hr.js +49 -0
- data/web/assets/javascripts/locales/jquery.timeago.hu.js +18 -0
- data/web/assets/javascripts/locales/jquery.timeago.hy.js +18 -0
- data/web/assets/javascripts/locales/jquery.timeago.id.js +18 -0
- data/web/assets/javascripts/locales/jquery.timeago.it.js +16 -0
- data/web/assets/javascripts/locales/jquery.timeago.ja.js +19 -0
- data/web/assets/javascripts/locales/jquery.timeago.ko.js +17 -0
- data/web/assets/javascripts/locales/jquery.timeago.lt.js +20 -0
- data/web/assets/javascripts/locales/jquery.timeago.mk.js +20 -0
- data/web/assets/javascripts/locales/jquery.timeago.nl.js +20 -0
- data/web/assets/javascripts/locales/jquery.timeago.no.js +18 -0
- data/web/assets/javascripts/locales/jquery.timeago.pl.js +31 -0
- data/web/assets/javascripts/locales/jquery.timeago.pt-br.js +16 -0
- data/web/assets/javascripts/locales/jquery.timeago.pt.js +16 -0
- data/web/assets/javascripts/locales/jquery.timeago.ro.js +18 -0
- data/web/assets/javascripts/locales/jquery.timeago.rs.js +49 -0
- data/web/assets/javascripts/locales/jquery.timeago.ru.js +34 -0
- data/web/assets/javascripts/locales/jquery.timeago.sk.js +18 -0
- data/web/assets/javascripts/locales/jquery.timeago.sl.js +44 -0
- data/web/assets/javascripts/locales/jquery.timeago.sv.js +18 -0
- data/web/assets/javascripts/locales/jquery.timeago.th.js +20 -0
- data/web/assets/javascripts/locales/jquery.timeago.tr.js +16 -0
- data/web/assets/javascripts/locales/jquery.timeago.uk.js +34 -0
- data/web/assets/javascripts/locales/jquery.timeago.uz.js +19 -0
- data/web/assets/javascripts/locales/jquery.timeago.zh-cn.js +20 -0
- data/web/assets/javascripts/locales/jquery.timeago.zh-tw.js +20 -0
- data/web/assets/stylesheets/application.css +754 -0
- data/web/assets/stylesheets/bootstrap.css +9 -0
- data/web/locales/cs.yml +78 -0
- data/web/locales/da.yml +68 -0
- data/web/locales/de.yml +69 -0
- data/web/locales/el.yml +68 -0
- data/web/locales/en.yml +79 -0
- data/web/locales/es.yml +69 -0
- data/web/locales/fr.yml +78 -0
- data/web/locales/hi.yml +75 -0
- data/web/locales/it.yml +69 -0
- data/web/locales/ja.yml +78 -0
- data/web/locales/ko.yml +68 -0
- data/web/locales/nb.yml +77 -0
- data/web/locales/nl.yml +68 -0
- data/web/locales/pl.yml +59 -0
- data/web/locales/pt-br.yml +68 -0
- data/web/locales/pt.yml +67 -0
- data/web/locales/ru.yml +78 -0
- data/web/locales/sv.yml +68 -0
- data/web/locales/ta.yml +75 -0
- data/web/locales/uk.yml +76 -0
- data/web/locales/zh-cn.yml +68 -0
- data/web/locales/zh-tw.yml +68 -0
- data/web/views/_footer.erb +17 -0
- data/web/views/_job_info.erb +88 -0
- data/web/views/_nav.erb +66 -0
- data/web/views/_paging.erb +23 -0
- data/web/views/_poll_js.erb +5 -0
- data/web/views/_poll_link.erb +7 -0
- data/web/views/_status.erb +4 -0
- data/web/views/_summary.erb +40 -0
- data/web/views/busy.erb +94 -0
- data/web/views/dashboard.erb +75 -0
- data/web/views/dead.erb +34 -0
- data/web/views/layout.erb +32 -0
- data/web/views/morgue.erb +71 -0
- data/web/views/queue.erb +45 -0
- data/web/views/queues.erb +28 -0
- data/web/views/retries.erb +74 -0
- data/web/views/retry.erb +34 -0
- data/web/views/scheduled.erb +54 -0
- data/web/views/scheduled_job_info.erb +8 -0
- metadata +408 -0
@@ -0,0 +1,106 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
begin
|
3
|
+
require 'active_support/core_ext/class/attribute'
|
4
|
+
rescue LoadError
|
5
|
+
|
6
|
+
# A dumbed down version of ActiveSupport's
|
7
|
+
# Class#class_attribute helper.
|
8
|
+
class Class
|
9
|
+
def class_attribute(*attrs)
|
10
|
+
instance_writer = true
|
11
|
+
|
12
|
+
attrs.each do |name|
|
13
|
+
class_eval <<-RUBY, __FILE__, __LINE__ + 1
|
14
|
+
def self.#{name}() nil end
|
15
|
+
def self.#{name}?() !!#{name} end
|
16
|
+
|
17
|
+
def self.#{name}=(val)
|
18
|
+
singleton_class.class_eval do
|
19
|
+
define_method(:#{name}) { val }
|
20
|
+
end
|
21
|
+
|
22
|
+
if singleton_class?
|
23
|
+
class_eval do
|
24
|
+
def #{name}
|
25
|
+
defined?(@#{name}) ? @#{name} : singleton_class.#{name}
|
26
|
+
end
|
27
|
+
end
|
28
|
+
end
|
29
|
+
val
|
30
|
+
end
|
31
|
+
|
32
|
+
def #{name}
|
33
|
+
defined?(@#{name}) ? @#{name} : self.class.#{name}
|
34
|
+
end
|
35
|
+
|
36
|
+
def #{name}?
|
37
|
+
!!#{name}
|
38
|
+
end
|
39
|
+
RUBY
|
40
|
+
|
41
|
+
attr_writer name if instance_writer
|
42
|
+
end
|
43
|
+
end
|
44
|
+
|
45
|
+
private
|
46
|
+
def singleton_class?
|
47
|
+
ancestors.first != self
|
48
|
+
end
|
49
|
+
end
|
50
|
+
end
|
51
|
+
|
52
|
+
begin
|
53
|
+
require 'active_support/core_ext/hash/keys'
|
54
|
+
require 'active_support/core_ext/hash/deep_merge'
|
55
|
+
rescue LoadError
|
56
|
+
class Hash
|
57
|
+
def stringify_keys
|
58
|
+
keys.each do |key|
|
59
|
+
self[key.to_s] = delete(key)
|
60
|
+
end
|
61
|
+
self
|
62
|
+
end if !{}.respond_to?(:stringify_keys)
|
63
|
+
|
64
|
+
def symbolize_keys
|
65
|
+
keys.each do |key|
|
66
|
+
self[(key.to_sym rescue key) || key] = delete(key)
|
67
|
+
end
|
68
|
+
self
|
69
|
+
end if !{}.respond_to?(:symbolize_keys)
|
70
|
+
|
71
|
+
def deep_merge(other_hash, &block)
|
72
|
+
dup.deep_merge!(other_hash, &block)
|
73
|
+
end if !{}.respond_to?(:deep_merge)
|
74
|
+
|
75
|
+
def deep_merge!(other_hash, &block)
|
76
|
+
other_hash.each_pair do |k,v|
|
77
|
+
tv = self[k]
|
78
|
+
if tv.is_a?(Hash) && v.is_a?(Hash)
|
79
|
+
self[k] = tv.deep_merge(v, &block)
|
80
|
+
else
|
81
|
+
self[k] = block && tv ? block.call(k, tv, v) : v
|
82
|
+
end
|
83
|
+
end
|
84
|
+
self
|
85
|
+
end if !{}.respond_to?(:deep_merge!)
|
86
|
+
end
|
87
|
+
end
|
88
|
+
|
89
|
+
begin
|
90
|
+
require 'active_support/core_ext/string/inflections'
|
91
|
+
rescue LoadError
|
92
|
+
class String
|
93
|
+
def constantize
|
94
|
+
names = self.split('::')
|
95
|
+
names.shift if names.empty? || names.first.empty?
|
96
|
+
|
97
|
+
constant = Object
|
98
|
+
names.each do |name|
|
99
|
+
constant = constant.const_defined?(name) ? constant.const_get(name) : constant.const_missing(name)
|
100
|
+
end
|
101
|
+
constant
|
102
|
+
end
|
103
|
+
end if !"".respond_to?(:constantize)
|
104
|
+
end
|
105
|
+
|
106
|
+
|
@@ -0,0 +1,31 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
require 'sidekiq'
|
3
|
+
|
4
|
+
module Sidekiq
|
5
|
+
module ExceptionHandler
|
6
|
+
|
7
|
+
class Logger
|
8
|
+
def call(ex, ctxHash)
|
9
|
+
Sidekiq.logger.warn(Sidekiq.dump_json(ctxHash)) if !ctxHash.empty?
|
10
|
+
Sidekiq.logger.warn "#{ex.class.name}: #{ex.message}"
|
11
|
+
Sidekiq.logger.warn ex.backtrace.join("\n") unless ex.backtrace.nil?
|
12
|
+
end
|
13
|
+
|
14
|
+
# Set up default handler which just logs the error
|
15
|
+
Sidekiq.error_handlers << Sidekiq::ExceptionHandler::Logger.new
|
16
|
+
end
|
17
|
+
|
18
|
+
def handle_exception(ex, ctxHash={})
|
19
|
+
Sidekiq.error_handlers.each do |handler|
|
20
|
+
begin
|
21
|
+
handler.call(ex, ctxHash)
|
22
|
+
rescue => ex
|
23
|
+
Sidekiq.logger.error "!!! ERROR HANDLER THREW AN ERROR !!!"
|
24
|
+
Sidekiq.logger.error ex
|
25
|
+
Sidekiq.logger.error ex.backtrace.join("\n") unless ex.backtrace.nil?
|
26
|
+
end
|
27
|
+
end
|
28
|
+
end
|
29
|
+
|
30
|
+
end
|
31
|
+
end
|
@@ -0,0 +1,57 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
require 'sidekiq/extensions/generic_proxy'
|
3
|
+
|
4
|
+
module Sidekiq
|
5
|
+
module Extensions
|
6
|
+
##
|
7
|
+
# Adds 'delay', 'delay_for' and `delay_until` methods to ActionMailer to offload arbitrary email
|
8
|
+
# delivery to Sidekiq. Example:
|
9
|
+
#
|
10
|
+
# UserMailer.delay.send_welcome_email(new_user)
|
11
|
+
# UserMailer.delay_for(5.days).send_welcome_email(new_user)
|
12
|
+
# UserMailer.delay_until(5.days.from_now).send_welcome_email(new_user)
|
13
|
+
class DelayedMailer
|
14
|
+
include Sidekiq::Worker
|
15
|
+
|
16
|
+
def perform(yml)
|
17
|
+
(target, method_name, args) = YAML.load(yml)
|
18
|
+
msg = target.public_send(method_name, *args)
|
19
|
+
# The email method can return nil, which causes ActionMailer to return
|
20
|
+
# an undeliverable empty message.
|
21
|
+
if msg
|
22
|
+
deliver(msg)
|
23
|
+
else
|
24
|
+
raise "#{target.name}##{method_name} returned an undeliverable mail object"
|
25
|
+
end
|
26
|
+
end
|
27
|
+
|
28
|
+
private
|
29
|
+
|
30
|
+
def deliver(msg)
|
31
|
+
if msg.respond_to?(:deliver_now)
|
32
|
+
# Rails 4.2/5.0
|
33
|
+
msg.deliver_now
|
34
|
+
else
|
35
|
+
# Rails 3.2/4.0/4.1
|
36
|
+
msg.deliver
|
37
|
+
end
|
38
|
+
end
|
39
|
+
end
|
40
|
+
|
41
|
+
module ActionMailer
|
42
|
+
def sidekiq_delay(options={})
|
43
|
+
Proxy.new(DelayedMailer, self, options)
|
44
|
+
end
|
45
|
+
def sidekiq_delay_for(interval, options={})
|
46
|
+
Proxy.new(DelayedMailer, self, options.merge('at' => Time.now.to_f + interval.to_f))
|
47
|
+
end
|
48
|
+
def sidekiq_delay_until(timestamp, options={})
|
49
|
+
Proxy.new(DelayedMailer, self, options.merge('at' => timestamp.to_f))
|
50
|
+
end
|
51
|
+
alias_method :delay, :sidekiq_delay
|
52
|
+
alias_method :delay_for, :sidekiq_delay_for
|
53
|
+
alias_method :delay_until, :sidekiq_delay_until
|
54
|
+
end
|
55
|
+
|
56
|
+
end
|
57
|
+
end
|
@@ -0,0 +1,40 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
require 'sidekiq/extensions/generic_proxy'
|
3
|
+
|
4
|
+
module Sidekiq
|
5
|
+
module Extensions
|
6
|
+
##
|
7
|
+
# Adds 'delay', 'delay_for' and `delay_until` methods to ActiveRecord to offload instance method
|
8
|
+
# execution to Sidekiq. Examples:
|
9
|
+
#
|
10
|
+
# User.recent_signups.each { |user| user.delay.mark_as_awesome }
|
11
|
+
#
|
12
|
+
# Please note, this is not recommended as this will serialize the entire
|
13
|
+
# object to Redis. Your Sidekiq jobs should pass IDs, not entire instances.
|
14
|
+
# This is here for backwards compatibility with Delayed::Job only.
|
15
|
+
class DelayedModel
|
16
|
+
include Sidekiq::Worker
|
17
|
+
|
18
|
+
def perform(yml)
|
19
|
+
(target, method_name, args) = YAML.load(yml)
|
20
|
+
target.__send__(method_name, *args)
|
21
|
+
end
|
22
|
+
end
|
23
|
+
|
24
|
+
module ActiveRecord
|
25
|
+
def sidekiq_delay(options={})
|
26
|
+
Proxy.new(DelayedModel, self, options)
|
27
|
+
end
|
28
|
+
def sidekiq_delay_for(interval, options={})
|
29
|
+
Proxy.new(DelayedModel, self, options.merge('at' => Time.now.to_f + interval.to_f))
|
30
|
+
end
|
31
|
+
def sidekiq_delay_until(timestamp, options={})
|
32
|
+
Proxy.new(DelayedModel, self, options.merge('at' => timestamp.to_f))
|
33
|
+
end
|
34
|
+
alias_method :delay, :sidekiq_delay
|
35
|
+
alias_method :delay_for, :sidekiq_delay_for
|
36
|
+
alias_method :delay_until, :sidekiq_delay_until
|
37
|
+
end
|
38
|
+
|
39
|
+
end
|
40
|
+
end
|
@@ -0,0 +1,40 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
require 'sidekiq/extensions/generic_proxy'
|
3
|
+
|
4
|
+
module Sidekiq
|
5
|
+
module Extensions
|
6
|
+
##
|
7
|
+
# Adds 'delay', 'delay_for' and `delay_until` methods to all Classes to offload class method
|
8
|
+
# execution to Sidekiq. Examples:
|
9
|
+
#
|
10
|
+
# User.delay.delete_inactive
|
11
|
+
# Wikipedia.delay.download_changes_for(Date.today)
|
12
|
+
#
|
13
|
+
class DelayedClass
|
14
|
+
include Sidekiq::Worker
|
15
|
+
|
16
|
+
def perform(yml)
|
17
|
+
(target, method_name, args) = YAML.load(yml)
|
18
|
+
target.__send__(method_name, *args)
|
19
|
+
end
|
20
|
+
end
|
21
|
+
|
22
|
+
module Klass
|
23
|
+
def sidekiq_delay(options={})
|
24
|
+
Proxy.new(DelayedClass, self, options)
|
25
|
+
end
|
26
|
+
def sidekiq_delay_for(interval, options={})
|
27
|
+
Proxy.new(DelayedClass, self, options.merge('at' => Time.now.to_f + interval.to_f))
|
28
|
+
end
|
29
|
+
def sidekiq_delay_until(timestamp, options={})
|
30
|
+
Proxy.new(DelayedClass, self, options.merge('at' => timestamp.to_f))
|
31
|
+
end
|
32
|
+
alias_method :delay, :sidekiq_delay
|
33
|
+
alias_method :delay_for, :sidekiq_delay_for
|
34
|
+
alias_method :delay_until, :sidekiq_delay_until
|
35
|
+
end
|
36
|
+
|
37
|
+
end
|
38
|
+
end
|
39
|
+
|
40
|
+
Module.__send__(:include, Sidekiq::Extensions::Klass) unless defined?(::Rails)
|
@@ -0,0 +1,25 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
require 'yaml'
|
3
|
+
|
4
|
+
module Sidekiq
|
5
|
+
module Extensions
|
6
|
+
class Proxy < BasicObject
|
7
|
+
def initialize(performable, target, options={})
|
8
|
+
@performable = performable
|
9
|
+
@target = target
|
10
|
+
@opts = options
|
11
|
+
end
|
12
|
+
|
13
|
+
def method_missing(name, *args)
|
14
|
+
# Sidekiq has a limitation in that its message must be JSON.
|
15
|
+
# JSON can't round trip real Ruby objects so we use YAML to
|
16
|
+
# serialize the objects to a String. The YAML will be converted
|
17
|
+
# to JSON and then deserialized on the other side back into a
|
18
|
+
# Ruby object.
|
19
|
+
obj = [@target, name, args]
|
20
|
+
@performable.client_push({ 'class' => @performable, 'args' => [::YAML.dump(obj)] }.merge(@opts))
|
21
|
+
end
|
22
|
+
end
|
23
|
+
|
24
|
+
end
|
25
|
+
end
|
@@ -0,0 +1,81 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
require 'sidekiq'
|
3
|
+
|
4
|
+
module Sidekiq
|
5
|
+
class BasicFetch
|
6
|
+
# We want the fetch operation to timeout every few seconds so the thread
|
7
|
+
# can check if the process is shutting down.
|
8
|
+
TIMEOUT = 2
|
9
|
+
|
10
|
+
UnitOfWork = Struct.new(:queue, :job) do
|
11
|
+
def acknowledge
|
12
|
+
# nothing to do
|
13
|
+
end
|
14
|
+
|
15
|
+
def queue_name
|
16
|
+
queue.sub(/.*queue:/, ''.freeze)
|
17
|
+
end
|
18
|
+
|
19
|
+
def requeue
|
20
|
+
Sidekiq.redis do |conn|
|
21
|
+
conn.rpush("queue:#{queue_name}", job)
|
22
|
+
end
|
23
|
+
end
|
24
|
+
end
|
25
|
+
|
26
|
+
def initialize(options)
|
27
|
+
@strictly_ordered_queues = !!options[:strict]
|
28
|
+
@queues = options[:queues].map { |q| "queue:#{q}" }
|
29
|
+
if @strictly_ordered_queues
|
30
|
+
@queues = @queues.uniq
|
31
|
+
@queues << TIMEOUT
|
32
|
+
end
|
33
|
+
end
|
34
|
+
|
35
|
+
def retrieve_work
|
36
|
+
work = Sidekiq.redis { |conn| conn.brpop(*queues_cmd) }
|
37
|
+
UnitOfWork.new(*work) if work
|
38
|
+
end
|
39
|
+
|
40
|
+
# Creating the Redis#brpop command takes into account any
|
41
|
+
# configured queue weights. By default Redis#brpop returns
|
42
|
+
# data from the first queue that has pending elements. We
|
43
|
+
# recreate the queue command each time we invoke Redis#brpop
|
44
|
+
# to honor weights and avoid queue starvation.
|
45
|
+
def queues_cmd
|
46
|
+
if @strictly_ordered_queues
|
47
|
+
@queues
|
48
|
+
else
|
49
|
+
queues = @queues.shuffle.uniq
|
50
|
+
queues << TIMEOUT
|
51
|
+
queues
|
52
|
+
end
|
53
|
+
end
|
54
|
+
|
55
|
+
|
56
|
+
# By leaving this as a class method, it can be pluggable and used by the Manager actor. Making it
|
57
|
+
# an instance method will make it async to the Fetcher actor
|
58
|
+
def self.bulk_requeue(inprogress, options)
|
59
|
+
return if inprogress.empty?
|
60
|
+
|
61
|
+
Sidekiq.logger.debug { "Re-queueing terminated jobs" }
|
62
|
+
jobs_to_requeue = {}
|
63
|
+
inprogress.each do |unit_of_work|
|
64
|
+
jobs_to_requeue[unit_of_work.queue_name] ||= []
|
65
|
+
jobs_to_requeue[unit_of_work.queue_name] << unit_of_work.job
|
66
|
+
end
|
67
|
+
|
68
|
+
Sidekiq.redis do |conn|
|
69
|
+
conn.pipelined do
|
70
|
+
jobs_to_requeue.each do |queue, jobs|
|
71
|
+
conn.rpush("queue:#{queue}", jobs)
|
72
|
+
end
|
73
|
+
end
|
74
|
+
end
|
75
|
+
Sidekiq.logger.info("Pushed #{inprogress.size} jobs back to Redis")
|
76
|
+
rescue => ex
|
77
|
+
Sidekiq.logger.warn("Failed to requeue #{inprogress.size} jobs: #{ex.message}")
|
78
|
+
end
|
79
|
+
|
80
|
+
end
|
81
|
+
end
|
@@ -0,0 +1,160 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
# encoding: utf-8
|
3
|
+
require 'sidekiq/manager'
|
4
|
+
require 'sidekiq/fetch'
|
5
|
+
require 'sidekiq/scheduled'
|
6
|
+
|
7
|
+
module Sidekiq
|
8
|
+
# The Launcher is a very simple Actor whose job is to
|
9
|
+
# start, monitor and stop the core Actors in Sidekiq.
|
10
|
+
# If any of these actors die, the Sidekiq process exits
|
11
|
+
# immediately.
|
12
|
+
class Launcher
|
13
|
+
include Util
|
14
|
+
|
15
|
+
attr_accessor :manager, :poller, :fetcher
|
16
|
+
|
17
|
+
def initialize(options)
|
18
|
+
@manager = Sidekiq::Manager.new(options)
|
19
|
+
@poller = Sidekiq::Scheduled::Poller.new
|
20
|
+
@done = false
|
21
|
+
@options = options
|
22
|
+
end
|
23
|
+
|
24
|
+
def run
|
25
|
+
@thread = safe_thread("heartbeat", &method(:start_heartbeat))
|
26
|
+
@poller.start
|
27
|
+
@manager.start
|
28
|
+
end
|
29
|
+
|
30
|
+
# Stops this instance from processing any more jobs,
|
31
|
+
#
|
32
|
+
def quiet
|
33
|
+
@done = true
|
34
|
+
@manager.quiet
|
35
|
+
@poller.terminate
|
36
|
+
end
|
37
|
+
|
38
|
+
# Shuts down the process. This method does not
|
39
|
+
# return until all work is complete and cleaned up.
|
40
|
+
# It can take up to the timeout to complete.
|
41
|
+
def stop
|
42
|
+
deadline = Time.now + @options[:timeout]
|
43
|
+
|
44
|
+
@done = true
|
45
|
+
@manager.quiet
|
46
|
+
@poller.terminate
|
47
|
+
|
48
|
+
@manager.stop(deadline)
|
49
|
+
|
50
|
+
# Requeue everything in case there was a worker who grabbed work while stopped
|
51
|
+
# This call is a no-op in Sidekiq but necessary for Sidekiq Pro.
|
52
|
+
strategy = (@options[:fetch] || Sidekiq::BasicFetch)
|
53
|
+
strategy.bulk_requeue([], @options)
|
54
|
+
|
55
|
+
clear_heartbeat
|
56
|
+
end
|
57
|
+
|
58
|
+
def stopping?
|
59
|
+
@done
|
60
|
+
end
|
61
|
+
|
62
|
+
private unless $TESTING
|
63
|
+
|
64
|
+
JVM_RESERVED_SIGNALS = ['USR1', 'USR2'] # Don't Process#kill if we get these signals via the API
|
65
|
+
|
66
|
+
def heartbeat(k, data, json)
|
67
|
+
results = Sidekiq::CLI::PROCTITLES.map {|x| x.(self, data) }
|
68
|
+
results.compact!
|
69
|
+
$0 = results.join(' ')
|
70
|
+
|
71
|
+
❤(k, json)
|
72
|
+
end
|
73
|
+
|
74
|
+
def ❤(key, json)
|
75
|
+
fails = procd = 0
|
76
|
+
begin
|
77
|
+
Processor::FAILURE.update {|curr| fails = curr; 0 }
|
78
|
+
Processor::PROCESSED.update {|curr| procd = curr; 0 }
|
79
|
+
|
80
|
+
workers_key = "#{key}:workers".freeze
|
81
|
+
nowdate = Time.now.utc.strftime("%Y-%m-%d".freeze)
|
82
|
+
Sidekiq.redis do |conn|
|
83
|
+
conn.pipelined do
|
84
|
+
conn.incrby("stat:processed".freeze, procd)
|
85
|
+
conn.incrby("stat:processed:#{nowdate}", procd)
|
86
|
+
conn.incrby("stat:failed".freeze, fails)
|
87
|
+
conn.incrby("stat:failed:#{nowdate}", fails)
|
88
|
+
conn.del(workers_key)
|
89
|
+
Processor::WORKER_STATE.each_pair do |tid, hash|
|
90
|
+
conn.hset(workers_key, tid, Sidekiq.dump_json(hash))
|
91
|
+
end
|
92
|
+
conn.expire(workers_key, 60)
|
93
|
+
end
|
94
|
+
end
|
95
|
+
fails = procd = 0
|
96
|
+
|
97
|
+
_, _, _, msg = Sidekiq.redis do |conn|
|
98
|
+
conn.pipelined do
|
99
|
+
conn.sadd('processes', key)
|
100
|
+
conn.hmset(key, 'info', json, 'busy', Processor::WORKER_STATE.size, 'beat', Time.now.to_f, 'quiet', @done)
|
101
|
+
conn.expire(key, 60)
|
102
|
+
conn.rpop("#{key}-signals")
|
103
|
+
end
|
104
|
+
end
|
105
|
+
|
106
|
+
return unless msg
|
107
|
+
|
108
|
+
if JVM_RESERVED_SIGNALS.include?(msg)
|
109
|
+
Sidekiq::CLI.instance.handle_signal(msg)
|
110
|
+
else
|
111
|
+
::Process.kill(msg, $$)
|
112
|
+
end
|
113
|
+
rescue => e
|
114
|
+
# ignore all redis/network issues
|
115
|
+
logger.error("heartbeat: #{e.message}")
|
116
|
+
# don't lose the counts if there was a network issue
|
117
|
+
Processor::PROCESSED.increment(procd)
|
118
|
+
Processor::FAILURE.increment(fails)
|
119
|
+
end
|
120
|
+
end
|
121
|
+
|
122
|
+
def start_heartbeat
|
123
|
+
k = identity
|
124
|
+
data = {
|
125
|
+
'hostname' => hostname,
|
126
|
+
'started_at' => Time.now.to_f,
|
127
|
+
'pid' => $$,
|
128
|
+
'tag' => @options[:tag] || '',
|
129
|
+
'concurrency' => @options[:concurrency],
|
130
|
+
'queues' => @options[:queues].uniq,
|
131
|
+
'labels' => @options[:labels],
|
132
|
+
'identity' => k,
|
133
|
+
}
|
134
|
+
# this data doesn't change so dump it to a string
|
135
|
+
# now so we don't need to dump it every heartbeat.
|
136
|
+
json = Sidekiq.dump_json(data)
|
137
|
+
|
138
|
+
while true
|
139
|
+
heartbeat(k, data, json)
|
140
|
+
sleep 5
|
141
|
+
end
|
142
|
+
Sidekiq.logger.info("Heartbeat stopping...")
|
143
|
+
end
|
144
|
+
|
145
|
+
def clear_heartbeat
|
146
|
+
# Remove record from Redis since we are shutting down.
|
147
|
+
# Note we don't stop the heartbeat thread; if the process
|
148
|
+
# doesn't actually exit, it'll reappear in the Web UI.
|
149
|
+
Sidekiq.redis do |conn|
|
150
|
+
conn.pipelined do
|
151
|
+
conn.srem('processes', identity)
|
152
|
+
conn.del("#{identity}:workers")
|
153
|
+
end
|
154
|
+
end
|
155
|
+
rescue
|
156
|
+
# best effort, ignore network errors
|
157
|
+
end
|
158
|
+
|
159
|
+
end
|
160
|
+
end
|