sidekiq 5.2.9 → 6.4.1

Sign up to get free protection for your applications and to get access to all the features.

Potentially problematic release.


This version of sidekiq might be problematic. Click here for more details.

Files changed (106) hide show
  1. checksums.yaml +4 -4
  2. data/Changes.md +318 -1
  3. data/LICENSE +3 -3
  4. data/README.md +23 -34
  5. data/bin/sidekiq +27 -3
  6. data/bin/sidekiqload +67 -61
  7. data/bin/sidekiqmon +8 -0
  8. data/lib/generators/sidekiq/job_generator.rb +57 -0
  9. data/lib/generators/sidekiq/templates/{worker.rb.erb → job.rb.erb} +2 -2
  10. data/lib/generators/sidekiq/templates/{worker_spec.rb.erb → job_spec.rb.erb} +1 -1
  11. data/lib/generators/sidekiq/templates/{worker_test.rb.erb → job_test.rb.erb} +1 -1
  12. data/lib/sidekiq/api.rb +335 -267
  13. data/lib/sidekiq/cli.rb +164 -182
  14. data/lib/sidekiq/client.rb +58 -61
  15. data/lib/sidekiq/delay.rb +7 -6
  16. data/lib/sidekiq/exception_handler.rb +10 -12
  17. data/lib/sidekiq/extensions/action_mailer.rb +13 -22
  18. data/lib/sidekiq/extensions/active_record.rb +13 -10
  19. data/lib/sidekiq/extensions/class_methods.rb +14 -11
  20. data/lib/sidekiq/extensions/generic_proxy.rb +6 -4
  21. data/lib/sidekiq/fetch.rb +40 -32
  22. data/lib/sidekiq/job.rb +13 -0
  23. data/lib/sidekiq/job_logger.rb +33 -7
  24. data/lib/sidekiq/job_retry.rb +70 -71
  25. data/lib/sidekiq/job_util.rb +65 -0
  26. data/lib/sidekiq/launcher.rb +161 -71
  27. data/lib/sidekiq/logger.rb +170 -0
  28. data/lib/sidekiq/manager.rb +17 -21
  29. data/lib/sidekiq/middleware/chain.rb +20 -8
  30. data/lib/sidekiq/middleware/current_attributes.rb +57 -0
  31. data/lib/sidekiq/middleware/i18n.rb +5 -7
  32. data/lib/sidekiq/monitor.rb +133 -0
  33. data/lib/sidekiq/paginator.rb +20 -16
  34. data/lib/sidekiq/processor.rb +71 -70
  35. data/lib/sidekiq/rails.rb +40 -37
  36. data/lib/sidekiq/redis_connection.rb +48 -48
  37. data/lib/sidekiq/scheduled.rb +62 -28
  38. data/lib/sidekiq/sd_notify.rb +149 -0
  39. data/lib/sidekiq/systemd.rb +24 -0
  40. data/lib/sidekiq/testing/inline.rb +2 -1
  41. data/lib/sidekiq/testing.rb +36 -27
  42. data/lib/sidekiq/util.rb +57 -15
  43. data/lib/sidekiq/version.rb +2 -1
  44. data/lib/sidekiq/web/action.rb +15 -11
  45. data/lib/sidekiq/web/application.rb +88 -75
  46. data/lib/sidekiq/web/csrf_protection.rb +180 -0
  47. data/lib/sidekiq/web/helpers.rb +109 -92
  48. data/lib/sidekiq/web/router.rb +23 -19
  49. data/lib/sidekiq/web.rb +61 -105
  50. data/lib/sidekiq/worker.rb +247 -105
  51. data/lib/sidekiq.rb +77 -44
  52. data/sidekiq.gemspec +23 -16
  53. data/web/assets/images/apple-touch-icon.png +0 -0
  54. data/web/assets/javascripts/application.js +83 -64
  55. data/web/assets/javascripts/dashboard.js +54 -73
  56. data/web/assets/stylesheets/application-dark.css +143 -0
  57. data/web/assets/stylesheets/application-rtl.css +0 -4
  58. data/web/assets/stylesheets/application.css +45 -232
  59. data/web/locales/ar.yml +8 -2
  60. data/web/locales/de.yml +14 -2
  61. data/web/locales/en.yml +6 -1
  62. data/web/locales/es.yml +18 -2
  63. data/web/locales/fr.yml +10 -3
  64. data/web/locales/ja.yml +7 -1
  65. data/web/locales/lt.yml +83 -0
  66. data/web/locales/pl.yml +4 -4
  67. data/web/locales/ru.yml +4 -0
  68. data/web/locales/vi.yml +83 -0
  69. data/web/views/_footer.erb +1 -1
  70. data/web/views/_job_info.erb +3 -2
  71. data/web/views/_poll_link.erb +2 -5
  72. data/web/views/_summary.erb +7 -7
  73. data/web/views/busy.erb +54 -20
  74. data/web/views/dashboard.erb +22 -14
  75. data/web/views/dead.erb +3 -3
  76. data/web/views/layout.erb +3 -1
  77. data/web/views/morgue.erb +9 -6
  78. data/web/views/queue.erb +19 -10
  79. data/web/views/queues.erb +10 -2
  80. data/web/views/retries.erb +11 -8
  81. data/web/views/retry.erb +3 -3
  82. data/web/views/scheduled.erb +5 -2
  83. metadata +34 -64
  84. data/.circleci/config.yml +0 -61
  85. data/.github/contributing.md +0 -32
  86. data/.github/issue_template.md +0 -11
  87. data/.gitignore +0 -15
  88. data/.travis.yml +0 -11
  89. data/3.0-Upgrade.md +0 -70
  90. data/4.0-Upgrade.md +0 -53
  91. data/5.0-Upgrade.md +0 -56
  92. data/COMM-LICENSE +0 -97
  93. data/Ent-Changes.md +0 -238
  94. data/Gemfile +0 -23
  95. data/Pro-2.0-Upgrade.md +0 -138
  96. data/Pro-3.0-Upgrade.md +0 -44
  97. data/Pro-4.0-Upgrade.md +0 -35
  98. data/Pro-Changes.md +0 -759
  99. data/Rakefile +0 -9
  100. data/bin/sidekiqctl +0 -20
  101. data/code_of_conduct.md +0 -50
  102. data/lib/generators/sidekiq/worker_generator.rb +0 -49
  103. data/lib/sidekiq/core_ext.rb +0 -1
  104. data/lib/sidekiq/ctl.rb +0 -221
  105. data/lib/sidekiq/logging.rb +0 -122
  106. data/lib/sidekiq/middleware/server/active_record.rb +0 -23
data/Rakefile DELETED
@@ -1,9 +0,0 @@
1
- require 'bundler/gem_tasks'
2
- require 'rake/testtask'
3
-
4
- Rake::TestTask.new(:test) do |test|
5
- test.warning = true
6
- test.pattern = 'test/**/test_*.rb'
7
- end
8
-
9
- task default: :test
data/bin/sidekiqctl DELETED
@@ -1,20 +0,0 @@
1
- #!/usr/bin/env ruby
2
-
3
- require 'fileutils'
4
- require 'sidekiq/api'
5
- require 'sidekiq/ctl'
6
-
7
- if ARGV[0] == 'status'
8
- Sidekiq::Ctl::Status.new.display(ARGV[1])
9
- else
10
- if ARGV.length < 2
11
- Sidekiq::Ctl.print_usage
12
- else
13
- stage = ARGV[0]
14
- pidfile = ARGV[1]
15
- timeout = ARGV[2].to_i
16
- timeout = Sidekiq::Ctl::DEFAULT_KILL_TIMEOUT if timeout == 0
17
-
18
- Sidekiq::Ctl.new(stage, pidfile, timeout)
19
- end
20
- end
data/code_of_conduct.md DELETED
@@ -1,50 +0,0 @@
1
- # Contributor Code of Conduct
2
-
3
- As contributors and maintainers of this project, and in the interest of
4
- fostering an open and welcoming community, we pledge to respect all people who
5
- contribute through reporting issues, posting feature requests, updating
6
- documentation, submitting pull requests or patches, and other activities.
7
-
8
- We are committed to making participation in this project a harassment-free
9
- experience for everyone, regardless of level of experience, gender, gender
10
- identity and expression, sexual orientation, disability, personal appearance,
11
- body size, race, ethnicity, age, religion, or nationality.
12
-
13
- Examples of unacceptable behavior by participants include:
14
-
15
- * The use of sexualized language or imagery
16
- * Personal attacks
17
- * Trolling or insulting/derogatory comments
18
- * Public or private harassment
19
- * Publishing other's private information, such as physical or electronic
20
- addresses, without explicit permission
21
- * Other unethical or unprofessional conduct
22
-
23
- Project maintainers have the right and responsibility to remove, edit, or
24
- reject comments, commits, code, wiki edits, issues, and other contributions
25
- that are not aligned to this Code of Conduct, or to ban temporarily or
26
- permanently any contributor for other behaviors that they deem inappropriate,
27
- threatening, offensive, or harmful.
28
-
29
- By adopting this Code of Conduct, project maintainers commit themselves to
30
- fairly and consistently applying these principles to every aspect of managing
31
- this project. Project maintainers who do not follow or enforce the Code of
32
- Conduct may be permanently removed from the project team.
33
-
34
- This Code of Conduct applies both within project spaces and in public spaces
35
- when an individual is representing the project or its community.
36
-
37
- Instances of abusive, harassing, or otherwise unacceptable behavior may be
38
- reported by contacting the project maintainer at mperham AT gmail.com. All
39
- complaints will be reviewed and investigated and will result in a response that
40
- is deemed necessary and appropriate to the circumstances. Maintainers are
41
- obligated to maintain confidentiality with regard to the reporter of an
42
- incident.
43
-
44
-
45
- This Code of Conduct is adapted from the [Contributor Covenant][homepage],
46
- version 1.3.0, available at
47
- [http://contributor-covenant.org/version/1/3/0/][version]
48
-
49
- [homepage]: http://contributor-covenant.org
50
- [version]: http://contributor-covenant.org/version/1/3/0/
@@ -1,49 +0,0 @@
1
- require 'rails/generators/named_base'
2
-
3
- module Sidekiq
4
- module Generators # :nodoc:
5
- class WorkerGenerator < ::Rails::Generators::NamedBase # :nodoc:
6
- desc 'This generator creates a Sidekiq Worker in app/workers and a corresponding test'
7
-
8
- check_class_collision suffix: 'Worker'
9
-
10
- def self.default_generator_root
11
- File.dirname(__FILE__)
12
- end
13
-
14
- def create_worker_file
15
- template 'worker.rb.erb', File.join('app/workers', class_path, "#{file_name}_worker.rb")
16
- end
17
-
18
- def create_test_file
19
- if defined?(RSpec)
20
- create_worker_spec
21
- else
22
- create_worker_test
23
- end
24
- end
25
-
26
- private
27
-
28
- def create_worker_spec
29
- template_file = File.join(
30
- 'spec/workers',
31
- class_path,
32
- "#{file_name}_worker_spec.rb"
33
- )
34
- template 'worker_spec.rb.erb', template_file
35
- end
36
-
37
- def create_worker_test
38
- template_file = File.join(
39
- 'test/workers',
40
- class_path,
41
- "#{file_name}_worker_test.rb"
42
- )
43
- template 'worker_test.rb.erb', template_file
44
- end
45
-
46
-
47
- end
48
- end
49
- end
@@ -1 +0,0 @@
1
- raise "no longer used, will be removed in 5.1"
data/lib/sidekiq/ctl.rb DELETED
@@ -1,221 +0,0 @@
1
- #!/usr/bin/env ruby
2
-
3
- require 'fileutils'
4
- require 'sidekiq/api'
5
-
6
- class Sidekiq::Ctl
7
- DEFAULT_KILL_TIMEOUT = 10
8
- CMD = File.basename($0)
9
-
10
- attr_reader :stage, :pidfile, :kill_timeout
11
-
12
- def self.print_usage
13
- puts "#{CMD} - control Sidekiq from the command line."
14
- puts
15
- puts "Usage: #{CMD} quiet <pidfile> <kill_timeout>"
16
- puts " #{CMD} stop <pidfile> <kill_timeout>"
17
- puts " #{CMD} status <section>"
18
- puts
19
- puts " <pidfile> is path to a pidfile"
20
- puts " <kill_timeout> is number of seconds to wait until Sidekiq exits"
21
- puts " (default: #{Sidekiq::Ctl::DEFAULT_KILL_TIMEOUT}), after which Sidekiq will be KILL'd"
22
- puts
23
- puts " <section> (optional) view a specific section of the status output"
24
- puts " Valid sections are: #{Sidekiq::Ctl::Status::VALID_SECTIONS.join(', ')}"
25
- puts
26
- puts "Be sure to set the kill_timeout LONGER than Sidekiq's -t timeout. If you want"
27
- puts "to wait 60 seconds for jobs to finish, use `sidekiq -t 60` and `sidekiqctl stop"
28
- puts " path_to_pidfile 61`"
29
- puts
30
- end
31
-
32
- def initialize(stage, pidfile, timeout)
33
- @stage = stage
34
- @pidfile = pidfile
35
- @kill_timeout = timeout
36
-
37
- done('No pidfile given', :error) if !pidfile
38
- done("Pidfile #{pidfile} does not exist", :warn) if !File.exist?(pidfile)
39
- done('Invalid pidfile content', :error) if pid == 0
40
-
41
- fetch_process
42
-
43
- begin
44
- send(stage)
45
- rescue NoMethodError
46
- done "Invalid command: #{stage}", :error
47
- end
48
- end
49
-
50
- def fetch_process
51
- Process.kill(0, pid)
52
- rescue Errno::ESRCH
53
- done "Process doesn't exist", :error
54
- # We were not allowed to send a signal, but the process must have existed
55
- # when Process.kill() was called.
56
- rescue Errno::EPERM
57
- return pid
58
- end
59
-
60
- def done(msg, error = nil)
61
- puts msg
62
- exit(exit_signal(error))
63
- end
64
-
65
- def exit_signal(error)
66
- (error == :error) ? 1 : 0
67
- end
68
-
69
- def pid
70
- @pid ||= File.read(pidfile).to_i
71
- end
72
-
73
- def quiet
74
- `kill -TSTP #{pid}`
75
- end
76
-
77
- def stop
78
- `kill -TERM #{pid}`
79
- kill_timeout.times do
80
- begin
81
- Process.kill(0, pid)
82
- rescue Errno::ESRCH
83
- FileUtils.rm_f pidfile
84
- done 'Sidekiq shut down gracefully.'
85
- rescue Errno::EPERM
86
- done 'Not permitted to shut down Sidekiq.'
87
- end
88
- sleep 1
89
- end
90
- `kill -9 #{pid}`
91
- FileUtils.rm_f pidfile
92
- done 'Sidekiq shut down forcefully.'
93
- end
94
- alias_method :shutdown, :stop
95
-
96
- class Status
97
- VALID_SECTIONS = %w[all version overview processes queues]
98
- def display(section = nil)
99
- section ||= 'all'
100
- unless VALID_SECTIONS.include? section
101
- puts "I don't know how to check the status of '#{section}'!"
102
- puts "Try one of these: #{VALID_SECTIONS.join(', ')}"
103
- return
104
- end
105
- send(section)
106
- rescue StandardError => e
107
- puts "Couldn't get status: #{e}"
108
- end
109
-
110
- def all
111
- version
112
- puts
113
- overview
114
- puts
115
- processes
116
- puts
117
- queues
118
- end
119
-
120
- def version
121
- puts "Sidekiq #{Sidekiq::VERSION}"
122
- puts Time.now
123
- end
124
-
125
- def overview
126
- puts '---- Overview ----'
127
- puts " Processed: #{delimit stats.processed}"
128
- puts " Failed: #{delimit stats.failed}"
129
- puts " Busy: #{delimit stats.workers_size}"
130
- puts " Enqueued: #{delimit stats.enqueued}"
131
- puts " Retries: #{delimit stats.retry_size}"
132
- puts " Scheduled: #{delimit stats.scheduled_size}"
133
- puts " Dead: #{delimit stats.dead_size}"
134
- end
135
-
136
- def processes
137
- puts "---- Processes (#{process_set.size}) ----"
138
- process_set.each_with_index do |process, index|
139
- puts "#{process['identity']} #{tags_for(process)}"
140
- puts " Started: #{Time.at(process['started_at'])} (#{time_ago(process['started_at'])})"
141
- puts " Threads: #{process['concurrency']} (#{process['busy']} busy)"
142
- puts " Queues: #{split_multiline(process['queues'].sort, pad: 11)}"
143
- puts '' unless (index+1) == process_set.size
144
- end
145
- end
146
-
147
- COL_PAD = 2
148
- def queues
149
- puts "---- Queues (#{queue_data.size}) ----"
150
- columns = {
151
- name: [:ljust, (['name'] + queue_data.map(&:name)).map(&:length).max + COL_PAD],
152
- size: [:rjust, (['size'] + queue_data.map(&:size)).map(&:length).max + COL_PAD],
153
- latency: [:rjust, (['latency'] + queue_data.map(&:latency)).map(&:length).max + COL_PAD]
154
- }
155
- columns.each { |col, (dir, width)| print col.to_s.upcase.public_send(dir, width) }
156
- puts
157
- queue_data.each do |q|
158
- columns.each do |col, (dir, width)|
159
- print q.send(col).public_send(dir, width)
160
- end
161
- puts
162
- end
163
- end
164
-
165
- private
166
-
167
- def delimit(number)
168
- number.to_s.reverse.scan(/.{1,3}/).join(',').reverse
169
- end
170
-
171
- def split_multiline(values, opts = {})
172
- return 'none' unless values
173
- pad = opts[:pad] || 0
174
- max_length = opts[:max_length] || (80 - pad)
175
- out = []
176
- line = ''
177
- values.each do |value|
178
- if (line.length + value.length) > max_length
179
- out << line
180
- line = ' ' * pad
181
- end
182
- line << value + ', '
183
- end
184
- out << line[0..-3]
185
- out.join("\n")
186
- end
187
-
188
- def tags_for(process)
189
- tags = [
190
- process['tag'],
191
- process['labels'],
192
- (process['quiet'] == 'true' ? 'quiet' : nil)
193
- ].flatten.compact
194
- tags.any? ? "[#{tags.join('] [')}]" : nil
195
- end
196
-
197
- def time_ago(timestamp)
198
- seconds = Time.now - Time.at(timestamp)
199
- return 'just now' if seconds < 60
200
- return 'a minute ago' if seconds < 120
201
- return "#{seconds.floor / 60} minutes ago" if seconds < 3600
202
- return 'an hour ago' if seconds < 7200
203
- "#{seconds.floor / 60 / 60} hours ago"
204
- end
205
-
206
- QUEUE_STRUCT = Struct.new(:name, :size, :latency)
207
- def queue_data
208
- @queue_data ||= Sidekiq::Queue.all.map do |q|
209
- QUEUE_STRUCT.new(q.name, q.size.to_s, sprintf('%#.2f', q.latency))
210
- end
211
- end
212
-
213
- def process_set
214
- @process_set ||= Sidekiq::ProcessSet.new
215
- end
216
-
217
- def stats
218
- @stats ||= Sidekiq::Stats.new
219
- end
220
- end
221
- end
@@ -1,122 +0,0 @@
1
- # frozen_string_literal: true
2
- require 'time'
3
- require 'logger'
4
- require 'fcntl'
5
-
6
- module Sidekiq
7
- module Logging
8
-
9
- class Pretty < Logger::Formatter
10
- SPACE = " "
11
-
12
- # Provide a call() method that returns the formatted message.
13
- def call(severity, time, program_name, message)
14
- "#{time.utc.iso8601(3)} #{::Process.pid} TID-#{Sidekiq::Logging.tid}#{context} #{severity}: #{message}\n"
15
- end
16
-
17
- def context
18
- c = Thread.current[:sidekiq_context]
19
- " #{c.join(SPACE)}" if c && c.any?
20
- end
21
- end
22
-
23
- class WithoutTimestamp < Pretty
24
- def call(severity, time, program_name, message)
25
- "#{::Process.pid} TID-#{Sidekiq::Logging.tid}#{context} #{severity}: #{message}\n"
26
- end
27
- end
28
-
29
- def self.tid
30
- Thread.current['sidekiq_tid'] ||= (Thread.current.object_id ^ ::Process.pid).to_s(36)
31
- end
32
-
33
- def self.job_hash_context(job_hash)
34
- # If we're using a wrapper class, like ActiveJob, use the "wrapped"
35
- # attribute to expose the underlying thing.
36
- klass = job_hash['wrapped'] || job_hash["class"]
37
- bid = job_hash['bid']
38
- "#{klass} JID-#{job_hash['jid']}#{" BID-#{bid}" if bid}"
39
- end
40
-
41
- def self.with_job_hash_context(job_hash, &block)
42
- with_context(job_hash_context(job_hash), &block)
43
- end
44
-
45
- def self.with_context(msg)
46
- Thread.current[:sidekiq_context] ||= []
47
- Thread.current[:sidekiq_context] << msg
48
- yield
49
- ensure
50
- Thread.current[:sidekiq_context].pop
51
- end
52
-
53
- def self.initialize_logger(log_target = STDOUT)
54
- oldlogger = defined?(@logger) ? @logger : nil
55
- @logger = Logger.new(log_target)
56
- @logger.level = Logger::INFO
57
- @logger.formatter = ENV['DYNO'] ? WithoutTimestamp.new : Pretty.new
58
- oldlogger.close if oldlogger && !$TESTING # don't want to close testing's STDOUT logging
59
- @logger
60
- end
61
-
62
- def self.logger
63
- defined?(@logger) ? @logger : initialize_logger
64
- end
65
-
66
- def self.logger=(log)
67
- @logger = (log ? log : Logger.new(File::NULL))
68
- end
69
-
70
- # This reopens ALL logfiles in the process that have been rotated
71
- # using logrotate(8) (without copytruncate) or similar tools.
72
- # A +File+ object is considered for reopening if it is:
73
- # 1) opened with the O_APPEND and O_WRONLY flags
74
- # 2) the current open file handle does not match its original open path
75
- # 3) unbuffered (as far as userspace buffering goes, not O_SYNC)
76
- # Returns the number of files reopened
77
- def self.reopen_logs
78
- to_reopen = []
79
- append_flags = File::WRONLY | File::APPEND
80
-
81
- ObjectSpace.each_object(File) do |fp|
82
- begin
83
- if !fp.closed? && fp.stat.file? && fp.sync && (fp.fcntl(Fcntl::F_GETFL) & append_flags) == append_flags
84
- to_reopen << fp
85
- end
86
- rescue IOError, Errno::EBADF
87
- end
88
- end
89
-
90
- nr = 0
91
- to_reopen.each do |fp|
92
- orig_st = begin
93
- fp.stat
94
- rescue IOError, Errno::EBADF
95
- next
96
- end
97
-
98
- begin
99
- b = File.stat(fp.path)
100
- next if orig_st.ino == b.ino && orig_st.dev == b.dev
101
- rescue Errno::ENOENT
102
- end
103
-
104
- begin
105
- File.open(fp.path, 'a') { |tmpfp| fp.reopen(tmpfp) }
106
- fp.sync = true
107
- nr += 1
108
- rescue IOError, Errno::EBADF
109
- # not much we can do...
110
- end
111
- end
112
- nr
113
- rescue RuntimeError => ex
114
- # RuntimeError: ObjectSpace is disabled; each_object will only work with Class, pass -X+O to enable
115
- puts "Unable to reopen logs: #{ex.message}"
116
- end
117
-
118
- def logger
119
- Sidekiq::Logging.logger
120
- end
121
- end
122
- end
@@ -1,23 +0,0 @@
1
- # frozen_string_literal: true
2
- module Sidekiq
3
- module Middleware
4
- module Server
5
- class ActiveRecord
6
-
7
- def initialize
8
- # With Rails 5+ we must use the Reloader **always**.
9
- # The reloader handles code loading and db connection management.
10
- if defined?(::Rails) && defined?(::Rails::VERSION) && ::Rails::VERSION::MAJOR >= 5
11
- raise ArgumentError, "Rails 5 no longer needs or uses the ActiveRecord middleware."
12
- end
13
- end
14
-
15
- def call(*args)
16
- yield
17
- ensure
18
- ::ActiveRecord::Base.clear_active_connections!
19
- end
20
- end
21
- end
22
- end
23
- end