sidekiq 4.2.10 → 6.5.7

Sign up to get free protection for your applications and to get access to all the features.

Potentially problematic release.


This version of sidekiq might be problematic. Click here for more details.

Files changed (131) hide show
  1. checksums.yaml +5 -5
  2. data/Changes.md +573 -1
  3. data/LICENSE +3 -3
  4. data/README.md +25 -34
  5. data/bin/sidekiq +27 -3
  6. data/bin/sidekiqload +81 -74
  7. data/bin/sidekiqmon +8 -0
  8. data/lib/generators/sidekiq/job_generator.rb +57 -0
  9. data/lib/generators/sidekiq/templates/{worker.rb.erb → job.rb.erb} +2 -2
  10. data/lib/generators/sidekiq/templates/job_spec.rb.erb +6 -0
  11. data/lib/generators/sidekiq/templates/{worker_test.rb.erb → job_test.rb.erb} +1 -1
  12. data/lib/sidekiq/api.rb +585 -285
  13. data/lib/sidekiq/cli.rb +256 -233
  14. data/lib/sidekiq/client.rb +86 -83
  15. data/lib/sidekiq/component.rb +65 -0
  16. data/lib/sidekiq/delay.rb +43 -0
  17. data/lib/sidekiq/extensions/action_mailer.rb +13 -22
  18. data/lib/sidekiq/extensions/active_record.rb +13 -10
  19. data/lib/sidekiq/extensions/class_methods.rb +14 -11
  20. data/lib/sidekiq/extensions/generic_proxy.rb +13 -5
  21. data/lib/sidekiq/fetch.rb +50 -40
  22. data/lib/sidekiq/job.rb +13 -0
  23. data/lib/sidekiq/job_logger.rb +51 -0
  24. data/lib/sidekiq/job_retry.rb +282 -0
  25. data/lib/sidekiq/job_util.rb +71 -0
  26. data/lib/sidekiq/launcher.rb +184 -90
  27. data/lib/sidekiq/logger.rb +156 -0
  28. data/lib/sidekiq/manager.rb +43 -45
  29. data/lib/sidekiq/metrics/deploy.rb +47 -0
  30. data/lib/sidekiq/metrics/query.rb +153 -0
  31. data/lib/sidekiq/metrics/shared.rb +94 -0
  32. data/lib/sidekiq/metrics/tracking.rb +134 -0
  33. data/lib/sidekiq/middleware/chain.rb +102 -46
  34. data/lib/sidekiq/middleware/current_attributes.rb +63 -0
  35. data/lib/sidekiq/middleware/i18n.rb +7 -7
  36. data/lib/sidekiq/middleware/modules.rb +21 -0
  37. data/lib/sidekiq/monitor.rb +133 -0
  38. data/lib/sidekiq/paginator.rb +20 -16
  39. data/lib/sidekiq/processor.rb +176 -91
  40. data/lib/sidekiq/rails.rb +41 -96
  41. data/lib/sidekiq/redis_client_adapter.rb +154 -0
  42. data/lib/sidekiq/redis_connection.rb +117 -48
  43. data/lib/sidekiq/ring_buffer.rb +29 -0
  44. data/lib/sidekiq/scheduled.rb +134 -44
  45. data/lib/sidekiq/sd_notify.rb +149 -0
  46. data/lib/sidekiq/systemd.rb +24 -0
  47. data/lib/sidekiq/testing/inline.rb +6 -5
  48. data/lib/sidekiq/testing.rb +80 -61
  49. data/lib/sidekiq/transaction_aware_client.rb +45 -0
  50. data/lib/sidekiq/version.rb +2 -1
  51. data/lib/sidekiq/web/action.rb +15 -15
  52. data/lib/sidekiq/web/application.rb +129 -86
  53. data/lib/sidekiq/web/csrf_protection.rb +180 -0
  54. data/lib/sidekiq/web/helpers.rb +170 -83
  55. data/lib/sidekiq/web/router.rb +23 -19
  56. data/lib/sidekiq/web.rb +69 -109
  57. data/lib/sidekiq/worker.rb +290 -41
  58. data/lib/sidekiq.rb +185 -77
  59. data/sidekiq.gemspec +23 -27
  60. data/web/assets/images/apple-touch-icon.png +0 -0
  61. data/web/assets/javascripts/application.js +112 -61
  62. data/web/assets/javascripts/chart.min.js +13 -0
  63. data/web/assets/javascripts/chartjs-plugin-annotation.min.js +7 -0
  64. data/web/assets/javascripts/dashboard.js +70 -91
  65. data/web/assets/javascripts/graph.js +16 -0
  66. data/web/assets/javascripts/metrics.js +262 -0
  67. data/web/assets/stylesheets/application-dark.css +143 -0
  68. data/web/assets/stylesheets/application-rtl.css +242 -0
  69. data/web/assets/stylesheets/application.css +364 -144
  70. data/web/assets/stylesheets/bootstrap-rtl.min.css +9 -0
  71. data/web/assets/stylesheets/bootstrap.css +2 -2
  72. data/web/locales/ar.yml +87 -0
  73. data/web/locales/de.yml +14 -2
  74. data/web/locales/el.yml +43 -19
  75. data/web/locales/en.yml +15 -1
  76. data/web/locales/es.yml +22 -5
  77. data/web/locales/fa.yml +1 -0
  78. data/web/locales/fr.yml +10 -3
  79. data/web/locales/he.yml +79 -0
  80. data/web/locales/ja.yml +19 -4
  81. data/web/locales/lt.yml +83 -0
  82. data/web/locales/pl.yml +4 -4
  83. data/web/locales/pt-br.yml +27 -9
  84. data/web/locales/ru.yml +4 -0
  85. data/web/locales/ur.yml +80 -0
  86. data/web/locales/vi.yml +83 -0
  87. data/web/locales/zh-cn.yml +36 -11
  88. data/web/locales/zh-tw.yml +32 -7
  89. data/web/views/_footer.erb +5 -2
  90. data/web/views/_job_info.erb +3 -2
  91. data/web/views/_nav.erb +5 -19
  92. data/web/views/_paging.erb +1 -1
  93. data/web/views/_poll_link.erb +2 -5
  94. data/web/views/_summary.erb +7 -7
  95. data/web/views/busy.erb +62 -24
  96. data/web/views/dashboard.erb +24 -15
  97. data/web/views/dead.erb +3 -3
  98. data/web/views/layout.erb +14 -3
  99. data/web/views/metrics.erb +69 -0
  100. data/web/views/metrics_for_job.erb +87 -0
  101. data/web/views/morgue.erb +9 -6
  102. data/web/views/queue.erb +26 -12
  103. data/web/views/queues.erb +12 -2
  104. data/web/views/retries.erb +14 -7
  105. data/web/views/retry.erb +3 -3
  106. data/web/views/scheduled.erb +7 -4
  107. metadata +66 -206
  108. data/.github/contributing.md +0 -32
  109. data/.github/issue_template.md +0 -9
  110. data/.gitignore +0 -12
  111. data/.travis.yml +0 -18
  112. data/3.0-Upgrade.md +0 -70
  113. data/4.0-Upgrade.md +0 -53
  114. data/COMM-LICENSE +0 -95
  115. data/Ent-Changes.md +0 -173
  116. data/Gemfile +0 -29
  117. data/Pro-2.0-Upgrade.md +0 -138
  118. data/Pro-3.0-Upgrade.md +0 -44
  119. data/Pro-Changes.md +0 -628
  120. data/Rakefile +0 -12
  121. data/bin/sidekiqctl +0 -99
  122. data/code_of_conduct.md +0 -50
  123. data/lib/generators/sidekiq/templates/worker_spec.rb.erb +0 -6
  124. data/lib/generators/sidekiq/worker_generator.rb +0 -49
  125. data/lib/sidekiq/core_ext.rb +0 -119
  126. data/lib/sidekiq/exception_handler.rb +0 -31
  127. data/lib/sidekiq/logging.rb +0 -106
  128. data/lib/sidekiq/middleware/server/active_record.rb +0 -13
  129. data/lib/sidekiq/middleware/server/logging.rb +0 -31
  130. data/lib/sidekiq/middleware/server/retry_jobs.rb +0 -205
  131. data/lib/sidekiq/util.rb +0 -63
data/bin/sidekiqctl DELETED
@@ -1,99 +0,0 @@
1
- #!/usr/bin/env ruby
2
-
3
- require 'fileutils'
4
-
5
- class Sidekiqctl
6
- DEFAULT_KILL_TIMEOUT = 10
7
-
8
- attr_reader :stage, :pidfile, :kill_timeout
9
-
10
- def self.print_usage
11
- puts "#{File.basename($0)} - stop a Sidekiq process from the command line."
12
- puts
13
- puts "Usage: #{File.basename($0)} <command> <pidfile> <kill_timeout>"
14
- puts " where <command> is either 'quiet' or 'stop'"
15
- puts " <pidfile> is path to a pidfile"
16
- puts " <kill_timeout> is number of seconds to wait until Sidekiq exits"
17
- puts " (default: #{Sidekiqctl::DEFAULT_KILL_TIMEOUT}), after which Sidekiq will be KILL'd"
18
- puts
19
- puts "Be sure to set the kill_timeout LONGER than Sidekiq's -t timeout. If you want"
20
- puts "to wait 60 seconds for jobs to finish, use `sidekiq -t 60` and `sidekiqctl stop"
21
- puts " path_to_pidfile 61`"
22
- puts
23
- end
24
-
25
- def initialize(stage, pidfile, timeout)
26
- @stage = stage
27
- @pidfile = pidfile
28
- @kill_timeout = timeout
29
-
30
- done('No pidfile given', :error) if !pidfile
31
- done("Pidfile #{pidfile} does not exist", :warn) if !File.exist?(pidfile)
32
- done('Invalid pidfile content', :error) if pid == 0
33
-
34
- fetch_process
35
-
36
- begin
37
- send(stage)
38
- rescue NoMethodError
39
- done "Invalid command: #{stage}", :error
40
- end
41
- end
42
-
43
- def fetch_process
44
- Process.kill(0, pid)
45
- rescue Errno::ESRCH
46
- done "Process doesn't exist", :error
47
- # We were not allowed to send a signal, but the process must have existed
48
- # when Process.kill() was called.
49
- rescue Errno::EPERM
50
- return pid
51
- end
52
-
53
- def done(msg, error = nil)
54
- puts msg
55
- exit(exit_signal(error))
56
- end
57
-
58
- def exit_signal(error)
59
- (error == :error) ? 1 : 0
60
- end
61
-
62
- def pid
63
- @pid ||= File.read(pidfile).to_i
64
- end
65
-
66
- def quiet
67
- `kill -USR1 #{pid}`
68
- end
69
-
70
- def stop
71
- `kill -TERM #{pid}`
72
- kill_timeout.times do
73
- begin
74
- Process.kill(0, pid)
75
- rescue Errno::ESRCH
76
- FileUtils.rm_f pidfile
77
- done 'Sidekiq shut down gracefully.'
78
- rescue Errno::EPERM
79
- done 'Not permitted to shut down Sidekiq.'
80
- end
81
- sleep 1
82
- end
83
- `kill -9 #{pid}`
84
- FileUtils.rm_f pidfile
85
- done 'Sidekiq shut down forcefully.'
86
- end
87
- alias_method :shutdown, :stop
88
- end
89
-
90
- if ARGV.length < 2
91
- Sidekiqctl.print_usage
92
- else
93
- stage = ARGV[0]
94
- pidfile = ARGV[1]
95
- timeout = ARGV[2].to_i
96
- timeout = Sidekiqctl::DEFAULT_KILL_TIMEOUT if timeout == 0
97
-
98
- Sidekiqctl.new(stage, pidfile, timeout)
99
- end
data/code_of_conduct.md DELETED
@@ -1,50 +0,0 @@
1
- # Contributor Code of Conduct
2
-
3
- As contributors and maintainers of this project, and in the interest of
4
- fostering an open and welcoming community, we pledge to respect all people who
5
- contribute through reporting issues, posting feature requests, updating
6
- documentation, submitting pull requests or patches, and other activities.
7
-
8
- We are committed to making participation in this project a harassment-free
9
- experience for everyone, regardless of level of experience, gender, gender
10
- identity and expression, sexual orientation, disability, personal appearance,
11
- body size, race, ethnicity, age, religion, or nationality.
12
-
13
- Examples of unacceptable behavior by participants include:
14
-
15
- * The use of sexualized language or imagery
16
- * Personal attacks
17
- * Trolling or insulting/derogatory comments
18
- * Public or private harassment
19
- * Publishing other's private information, such as physical or electronic
20
- addresses, without explicit permission
21
- * Other unethical or unprofessional conduct
22
-
23
- Project maintainers have the right and responsibility to remove, edit, or
24
- reject comments, commits, code, wiki edits, issues, and other contributions
25
- that are not aligned to this Code of Conduct, or to ban temporarily or
26
- permanently any contributor for other behaviors that they deem inappropriate,
27
- threatening, offensive, or harmful.
28
-
29
- By adopting this Code of Conduct, project maintainers commit themselves to
30
- fairly and consistently applying these principles to every aspect of managing
31
- this project. Project maintainers who do not follow or enforce the Code of
32
- Conduct may be permanently removed from the project team.
33
-
34
- This Code of Conduct applies both within project spaces and in public spaces
35
- when an individual is representing the project or its community.
36
-
37
- Instances of abusive, harassing, or otherwise unacceptable behavior may be
38
- reported by contacting the project maintainer at mperham AT gmail.com. All
39
- complaints will be reviewed and investigated and will result in a response that
40
- is deemed necessary and appropriate to the circumstances. Maintainers are
41
- obligated to maintain confidentiality with regard to the reporter of an
42
- incident.
43
-
44
-
45
- This Code of Conduct is adapted from the [Contributor Covenant][homepage],
46
- version 1.3.0, available at
47
- [http://contributor-covenant.org/version/1/3/0/][version]
48
-
49
- [homepage]: http://contributor-covenant.org
50
- [version]: http://contributor-covenant.org/version/1/3/0/
@@ -1,6 +0,0 @@
1
- require 'rails_helper'
2
- <% module_namespacing do -%>
3
- RSpec.describe <%= class_name %>Worker, type: :worker do
4
- pending "add some examples to (or delete) #{__FILE__}"
5
- end
6
- <% end -%>
@@ -1,49 +0,0 @@
1
- require 'rails/generators/named_base'
2
-
3
- module Sidekiq
4
- module Generators # :nodoc:
5
- class WorkerGenerator < ::Rails::Generators::NamedBase # :nodoc:
6
- desc 'This generator creates a Sidekiq Worker in app/workers and a corresponding test'
7
-
8
- check_class_collision suffix: 'Worker'
9
-
10
- def self.default_generator_root
11
- File.dirname(__FILE__)
12
- end
13
-
14
- def create_worker_file
15
- template 'worker.rb.erb', File.join('app/workers', class_path, "#{file_name}_worker.rb")
16
- end
17
-
18
- def create_test_file
19
- if defined?(RSpec)
20
- create_worker_spec
21
- else
22
- create_worker_test
23
- end
24
- end
25
-
26
- private
27
-
28
- def create_worker_spec
29
- template_file = File.join(
30
- 'spec/workers',
31
- class_path,
32
- "#{file_name}_worker_spec.rb"
33
- )
34
- template 'worker_spec.rb.erb', template_file
35
- end
36
-
37
- def create_worker_test
38
- template_file = File.join(
39
- 'test/workers',
40
- class_path,
41
- "#{file_name}_worker_test.rb"
42
- )
43
- template 'worker_test.rb.erb', template_file
44
- end
45
-
46
-
47
- end
48
- end
49
- end
@@ -1,119 +0,0 @@
1
- # frozen_string_literal: true
2
- begin
3
- require 'active_support/core_ext/class/attribute'
4
- rescue LoadError
5
-
6
- # A dumbed down version of ActiveSupport's
7
- # Class#class_attribute helper.
8
- class Class
9
- def class_attribute(*attrs)
10
- instance_writer = true
11
-
12
- attrs.each do |name|
13
- class_eval <<-RUBY, __FILE__, __LINE__ + 1
14
- def self.#{name}() nil end
15
- def self.#{name}?() !!#{name} end
16
-
17
- def self.#{name}=(val)
18
- singleton_class.class_eval do
19
- define_method(:#{name}) { val }
20
- end
21
-
22
- if singleton_class?
23
- class_eval do
24
- def #{name}
25
- defined?(@#{name}) ? @#{name} : singleton_class.#{name}
26
- end
27
- end
28
- end
29
- val
30
- end
31
-
32
- def #{name}
33
- defined?(@#{name}) ? @#{name} : self.class.#{name}
34
- end
35
-
36
- def #{name}?
37
- !!#{name}
38
- end
39
- RUBY
40
-
41
- attr_writer name if instance_writer
42
- end
43
- end
44
-
45
- private
46
- def singleton_class?
47
- ancestors.first != self
48
- end
49
- end
50
- end
51
-
52
- begin
53
- require 'active_support/core_ext/hash/keys'
54
- require 'active_support/core_ext/hash/deep_merge'
55
- rescue LoadError
56
- class Hash
57
- def stringify_keys
58
- keys.each do |key|
59
- self[key.to_s] = delete(key)
60
- end
61
- self
62
- end if !{}.respond_to?(:stringify_keys)
63
-
64
- def symbolize_keys
65
- keys.each do |key|
66
- self[(key.to_sym rescue key) || key] = delete(key)
67
- end
68
- self
69
- end if !{}.respond_to?(:symbolize_keys)
70
-
71
- def deep_merge(other_hash, &block)
72
- dup.deep_merge!(other_hash, &block)
73
- end if !{}.respond_to?(:deep_merge)
74
-
75
- def deep_merge!(other_hash, &block)
76
- other_hash.each_pair do |k,v|
77
- tv = self[k]
78
- if tv.is_a?(Hash) && v.is_a?(Hash)
79
- self[k] = tv.deep_merge(v, &block)
80
- else
81
- self[k] = block && tv ? block.call(k, tv, v) : v
82
- end
83
- end
84
- self
85
- end if !{}.respond_to?(:deep_merge!)
86
- end
87
- end
88
-
89
- begin
90
- require 'active_support/core_ext/string/inflections'
91
- rescue LoadError
92
- class String
93
- def constantize
94
- names = self.split('::')
95
- names.shift if names.empty? || names.first.empty?
96
-
97
- constant = Object
98
- names.each do |name|
99
- constant = constant.const_defined?(name) ? constant.const_get(name) : constant.const_missing(name)
100
- end
101
- constant
102
- end
103
- end if !"".respond_to?(:constantize)
104
- end
105
-
106
-
107
- begin
108
- require 'active_support/core_ext/kernel/reporting'
109
- rescue LoadError
110
- module Kernel
111
- module_function
112
- def silence_warnings
113
- old_verbose, $VERBOSE = $VERBOSE, nil
114
- yield
115
- ensure
116
- $VERBOSE = old_verbose
117
- end
118
- end
119
- end
@@ -1,31 +0,0 @@
1
- # frozen_string_literal: true
2
- require 'sidekiq'
3
-
4
- module Sidekiq
5
- module ExceptionHandler
6
-
7
- class Logger
8
- def call(ex, ctxHash)
9
- Sidekiq.logger.warn(Sidekiq.dump_json(ctxHash)) if !ctxHash.empty?
10
- Sidekiq.logger.warn "#{ex.class.name}: #{ex.message}"
11
- Sidekiq.logger.warn ex.backtrace.join("\n") unless ex.backtrace.nil?
12
- end
13
-
14
- # Set up default handler which just logs the error
15
- Sidekiq.error_handlers << Sidekiq::ExceptionHandler::Logger.new
16
- end
17
-
18
- def handle_exception(ex, ctxHash={})
19
- Sidekiq.error_handlers.each do |handler|
20
- begin
21
- handler.call(ex, ctxHash)
22
- rescue => ex
23
- Sidekiq.logger.error "!!! ERROR HANDLER THREW AN ERROR !!!"
24
- Sidekiq.logger.error ex
25
- Sidekiq.logger.error ex.backtrace.join("\n") unless ex.backtrace.nil?
26
- end
27
- end
28
- end
29
-
30
- end
31
- end
@@ -1,106 +0,0 @@
1
- # frozen_string_literal: true
2
- require 'time'
3
- require 'logger'
4
- require 'fcntl'
5
-
6
- module Sidekiq
7
- module Logging
8
-
9
- class Pretty < Logger::Formatter
10
- SPACE = " "
11
-
12
- # Provide a call() method that returns the formatted message.
13
- def call(severity, time, program_name, message)
14
- "#{time.utc.iso8601(3)} #{::Process.pid} TID-#{Thread.current.object_id.to_s(36)}#{context} #{severity}: #{message}\n"
15
- end
16
-
17
- def context
18
- c = Thread.current[:sidekiq_context]
19
- " #{c.join(SPACE)}" if c && c.any?
20
- end
21
- end
22
-
23
- class WithoutTimestamp < Pretty
24
- def call(severity, time, program_name, message)
25
- "#{::Process.pid} TID-#{Thread.current.object_id.to_s(36)}#{context} #{severity}: #{message}\n"
26
- end
27
- end
28
-
29
- def self.with_context(msg)
30
- Thread.current[:sidekiq_context] ||= []
31
- Thread.current[:sidekiq_context] << msg
32
- yield
33
- ensure
34
- Thread.current[:sidekiq_context].pop
35
- end
36
-
37
- def self.initialize_logger(log_target = STDOUT)
38
- oldlogger = defined?(@logger) ? @logger : nil
39
- @logger = Logger.new(log_target)
40
- @logger.level = Logger::INFO
41
- @logger.formatter = ENV['DYNO'] ? WithoutTimestamp.new : Pretty.new
42
- oldlogger.close if oldlogger && !$TESTING # don't want to close testing's STDOUT logging
43
- @logger
44
- end
45
-
46
- def self.logger
47
- defined?(@logger) ? @logger : initialize_logger
48
- end
49
-
50
- def self.logger=(log)
51
- @logger = (log ? log : Logger.new(File::NULL))
52
- end
53
-
54
- # This reopens ALL logfiles in the process that have been rotated
55
- # using logrotate(8) (without copytruncate) or similar tools.
56
- # A +File+ object is considered for reopening if it is:
57
- # 1) opened with the O_APPEND and O_WRONLY flags
58
- # 2) the current open file handle does not match its original open path
59
- # 3) unbuffered (as far as userspace buffering goes, not O_SYNC)
60
- # Returns the number of files reopened
61
- def self.reopen_logs
62
- to_reopen = []
63
- append_flags = File::WRONLY | File::APPEND
64
-
65
- ObjectSpace.each_object(File) do |fp|
66
- begin
67
- if !fp.closed? && fp.stat.file? && fp.sync && (fp.fcntl(Fcntl::F_GETFL) & append_flags) == append_flags
68
- to_reopen << fp
69
- end
70
- rescue IOError, Errno::EBADF
71
- end
72
- end
73
-
74
- nr = 0
75
- to_reopen.each do |fp|
76
- orig_st = begin
77
- fp.stat
78
- rescue IOError, Errno::EBADF
79
- next
80
- end
81
-
82
- begin
83
- b = File.stat(fp.path)
84
- next if orig_st.ino == b.ino && orig_st.dev == b.dev
85
- rescue Errno::ENOENT
86
- end
87
-
88
- begin
89
- File.open(fp.path, 'a') { |tmpfp| fp.reopen(tmpfp) }
90
- fp.sync = true
91
- nr += 1
92
- rescue IOError, Errno::EBADF
93
- # not much we can do...
94
- end
95
- end
96
- nr
97
- rescue RuntimeError => ex
98
- # RuntimeError: ObjectSpace is disabled; each_object will only work with Class, pass -X+O to enable
99
- puts "Unable to reopen logs: #{ex.message}"
100
- end
101
-
102
- def logger
103
- Sidekiq::Logging.logger
104
- end
105
- end
106
- end
@@ -1,13 +0,0 @@
1
- module Sidekiq
2
- module Middleware
3
- module Server
4
- class ActiveRecord
5
- def call(*args)
6
- yield
7
- ensure
8
- ::ActiveRecord::Base.clear_active_connections!
9
- end
10
- end
11
- end
12
- end
13
- end
@@ -1,31 +0,0 @@
1
- module Sidekiq
2
- module Middleware
3
- module Server
4
- class Logging
5
-
6
- def call(worker, item, queue)
7
- begin
8
- start = Time.now
9
- logger.info("start".freeze)
10
- yield
11
- logger.info("done: #{elapsed(start)} sec")
12
- rescue Exception
13
- logger.info("fail: #{elapsed(start)} sec")
14
- raise
15
- end
16
- end
17
-
18
- private
19
-
20
- def elapsed(start)
21
- (Time.now - start).round(3)
22
- end
23
-
24
- def logger
25
- Sidekiq.logger
26
- end
27
- end
28
- end
29
- end
30
- end
31
-