sidekiq 5.2.6

Sign up to get free protection for your applications and to get access to all the features.

Potentially problematic release.


This version of sidekiq might be problematic. Click here for more details.

Files changed (118) hide show
  1. checksums.yaml +7 -0
  2. data/.github/contributing.md +32 -0
  3. data/.github/issue_template.md +11 -0
  4. data/.gitignore +15 -0
  5. data/.travis.yml +11 -0
  6. data/3.0-Upgrade.md +70 -0
  7. data/4.0-Upgrade.md +53 -0
  8. data/5.0-Upgrade.md +56 -0
  9. data/COMM-LICENSE +97 -0
  10. data/Changes.md +1529 -0
  11. data/Ent-Changes.md +238 -0
  12. data/Gemfile +23 -0
  13. data/LICENSE +9 -0
  14. data/Pro-2.0-Upgrade.md +138 -0
  15. data/Pro-3.0-Upgrade.md +44 -0
  16. data/Pro-4.0-Upgrade.md +35 -0
  17. data/Pro-Changes.md +759 -0
  18. data/README.md +109 -0
  19. data/Rakefile +9 -0
  20. data/bin/sidekiq +18 -0
  21. data/bin/sidekiqctl +20 -0
  22. data/bin/sidekiqload +149 -0
  23. data/code_of_conduct.md +50 -0
  24. data/lib/generators/sidekiq/templates/worker.rb.erb +9 -0
  25. data/lib/generators/sidekiq/templates/worker_spec.rb.erb +6 -0
  26. data/lib/generators/sidekiq/templates/worker_test.rb.erb +8 -0
  27. data/lib/generators/sidekiq/worker_generator.rb +49 -0
  28. data/lib/sidekiq.rb +237 -0
  29. data/lib/sidekiq/api.rb +940 -0
  30. data/lib/sidekiq/cli.rb +445 -0
  31. data/lib/sidekiq/client.rb +243 -0
  32. data/lib/sidekiq/core_ext.rb +1 -0
  33. data/lib/sidekiq/ctl.rb +221 -0
  34. data/lib/sidekiq/delay.rb +42 -0
  35. data/lib/sidekiq/exception_handler.rb +29 -0
  36. data/lib/sidekiq/extensions/action_mailer.rb +57 -0
  37. data/lib/sidekiq/extensions/active_record.rb +40 -0
  38. data/lib/sidekiq/extensions/class_methods.rb +40 -0
  39. data/lib/sidekiq/extensions/generic_proxy.rb +31 -0
  40. data/lib/sidekiq/fetch.rb +81 -0
  41. data/lib/sidekiq/job_logger.rb +25 -0
  42. data/lib/sidekiq/job_retry.rb +262 -0
  43. data/lib/sidekiq/launcher.rb +173 -0
  44. data/lib/sidekiq/logging.rb +122 -0
  45. data/lib/sidekiq/manager.rb +137 -0
  46. data/lib/sidekiq/middleware/chain.rb +150 -0
  47. data/lib/sidekiq/middleware/i18n.rb +42 -0
  48. data/lib/sidekiq/middleware/server/active_record.rb +23 -0
  49. data/lib/sidekiq/paginator.rb +43 -0
  50. data/lib/sidekiq/processor.rb +279 -0
  51. data/lib/sidekiq/rails.rb +58 -0
  52. data/lib/sidekiq/redis_connection.rb +144 -0
  53. data/lib/sidekiq/scheduled.rb +174 -0
  54. data/lib/sidekiq/testing.rb +333 -0
  55. data/lib/sidekiq/testing/inline.rb +29 -0
  56. data/lib/sidekiq/util.rb +66 -0
  57. data/lib/sidekiq/version.rb +4 -0
  58. data/lib/sidekiq/web.rb +213 -0
  59. data/lib/sidekiq/web/action.rb +89 -0
  60. data/lib/sidekiq/web/application.rb +353 -0
  61. data/lib/sidekiq/web/helpers.rb +325 -0
  62. data/lib/sidekiq/web/router.rb +100 -0
  63. data/lib/sidekiq/worker.rb +220 -0
  64. data/sidekiq.gemspec +21 -0
  65. data/web/assets/images/favicon.ico +0 -0
  66. data/web/assets/images/logo.png +0 -0
  67. data/web/assets/images/status.png +0 -0
  68. data/web/assets/javascripts/application.js +92 -0
  69. data/web/assets/javascripts/dashboard.js +315 -0
  70. data/web/assets/stylesheets/application-rtl.css +246 -0
  71. data/web/assets/stylesheets/application.css +1144 -0
  72. data/web/assets/stylesheets/bootstrap-rtl.min.css +9 -0
  73. data/web/assets/stylesheets/bootstrap.css +5 -0
  74. data/web/locales/ar.yml +81 -0
  75. data/web/locales/cs.yml +78 -0
  76. data/web/locales/da.yml +68 -0
  77. data/web/locales/de.yml +69 -0
  78. data/web/locales/el.yml +68 -0
  79. data/web/locales/en.yml +81 -0
  80. data/web/locales/es.yml +70 -0
  81. data/web/locales/fa.yml +80 -0
  82. data/web/locales/fr.yml +78 -0
  83. data/web/locales/he.yml +79 -0
  84. data/web/locales/hi.yml +75 -0
  85. data/web/locales/it.yml +69 -0
  86. data/web/locales/ja.yml +80 -0
  87. data/web/locales/ko.yml +68 -0
  88. data/web/locales/nb.yml +77 -0
  89. data/web/locales/nl.yml +68 -0
  90. data/web/locales/pl.yml +59 -0
  91. data/web/locales/pt-br.yml +68 -0
  92. data/web/locales/pt.yml +67 -0
  93. data/web/locales/ru.yml +78 -0
  94. data/web/locales/sv.yml +68 -0
  95. data/web/locales/ta.yml +75 -0
  96. data/web/locales/uk.yml +76 -0
  97. data/web/locales/ur.yml +80 -0
  98. data/web/locales/zh-cn.yml +68 -0
  99. data/web/locales/zh-tw.yml +68 -0
  100. data/web/views/_footer.erb +20 -0
  101. data/web/views/_job_info.erb +88 -0
  102. data/web/views/_nav.erb +52 -0
  103. data/web/views/_paging.erb +23 -0
  104. data/web/views/_poll_link.erb +7 -0
  105. data/web/views/_status.erb +4 -0
  106. data/web/views/_summary.erb +40 -0
  107. data/web/views/busy.erb +98 -0
  108. data/web/views/dashboard.erb +75 -0
  109. data/web/views/dead.erb +34 -0
  110. data/web/views/layout.erb +40 -0
  111. data/web/views/morgue.erb +75 -0
  112. data/web/views/queue.erb +46 -0
  113. data/web/views/queues.erb +30 -0
  114. data/web/views/retries.erb +80 -0
  115. data/web/views/retry.erb +34 -0
  116. data/web/views/scheduled.erb +54 -0
  117. data/web/views/scheduled_job_info.erb +8 -0
  118. metadata +229 -0
@@ -0,0 +1 @@
1
+ raise "no longer used, will be removed in 5.1"
@@ -0,0 +1,221 @@
1
+ #!/usr/bin/env ruby
2
+
3
+ require 'fileutils'
4
+ require 'sidekiq/api'
5
+
6
+ class Sidekiq::Ctl
7
+ DEFAULT_KILL_TIMEOUT = 10
8
+ CMD = File.basename($0)
9
+
10
+ attr_reader :stage, :pidfile, :kill_timeout
11
+
12
+ def self.print_usage
13
+ puts "#{CMD} - control Sidekiq from the command line."
14
+ puts
15
+ puts "Usage: #{CMD} quiet <pidfile> <kill_timeout>"
16
+ puts " #{CMD} stop <pidfile> <kill_timeout>"
17
+ puts " #{CMD} status <section>"
18
+ puts
19
+ puts " <pidfile> is path to a pidfile"
20
+ puts " <kill_timeout> is number of seconds to wait until Sidekiq exits"
21
+ puts " (default: #{Sidekiqctl::DEFAULT_KILL_TIMEOUT}), after which Sidekiq will be KILL'd"
22
+ puts
23
+ puts " <section> (optional) view a specific section of the status output"
24
+ puts " Valid sections are: #{Sidekiqctl::Status::VALID_SECTIONS.join(', ')}"
25
+ puts
26
+ puts "Be sure to set the kill_timeout LONGER than Sidekiq's -t timeout. If you want"
27
+ puts "to wait 60 seconds for jobs to finish, use `sidekiq -t 60` and `sidekiqctl stop"
28
+ puts " path_to_pidfile 61`"
29
+ puts
30
+ end
31
+
32
+ def initialize(stage, pidfile, timeout)
33
+ @stage = stage
34
+ @pidfile = pidfile
35
+ @kill_timeout = timeout
36
+
37
+ done('No pidfile given', :error) if !pidfile
38
+ done("Pidfile #{pidfile} does not exist", :warn) if !File.exist?(pidfile)
39
+ done('Invalid pidfile content', :error) if pid == 0
40
+
41
+ fetch_process
42
+
43
+ begin
44
+ send(stage)
45
+ rescue NoMethodError
46
+ done "Invalid command: #{stage}", :error
47
+ end
48
+ end
49
+
50
+ def fetch_process
51
+ Process.kill(0, pid)
52
+ rescue Errno::ESRCH
53
+ done "Process doesn't exist", :error
54
+ # We were not allowed to send a signal, but the process must have existed
55
+ # when Process.kill() was called.
56
+ rescue Errno::EPERM
57
+ return pid
58
+ end
59
+
60
+ def done(msg, error = nil)
61
+ puts msg
62
+ exit(exit_signal(error))
63
+ end
64
+
65
+ def exit_signal(error)
66
+ (error == :error) ? 1 : 0
67
+ end
68
+
69
+ def pid
70
+ @pid ||= File.read(pidfile).to_i
71
+ end
72
+
73
+ def quiet
74
+ `kill -TSTP #{pid}`
75
+ end
76
+
77
+ def stop
78
+ `kill -TERM #{pid}`
79
+ kill_timeout.times do
80
+ begin
81
+ Process.kill(0, pid)
82
+ rescue Errno::ESRCH
83
+ FileUtils.rm_f pidfile
84
+ done 'Sidekiq shut down gracefully.'
85
+ rescue Errno::EPERM
86
+ done 'Not permitted to shut down Sidekiq.'
87
+ end
88
+ sleep 1
89
+ end
90
+ `kill -9 #{pid}`
91
+ FileUtils.rm_f pidfile
92
+ done 'Sidekiq shut down forcefully.'
93
+ end
94
+ alias_method :shutdown, :stop
95
+
96
+ class Status
97
+ VALID_SECTIONS = %w[all version overview processes queues]
98
+ def display(section = nil)
99
+ section ||= 'all'
100
+ unless VALID_SECTIONS.include? section
101
+ puts "I don't know how to check the status of '#{section}'!"
102
+ puts "Try one of these: #{VALID_SECTIONS.join(', ')}"
103
+ return
104
+ end
105
+ send(section)
106
+ rescue StandardError => e
107
+ puts "Couldn't get status: #{e}"
108
+ end
109
+
110
+ def all
111
+ version
112
+ puts
113
+ overview
114
+ puts
115
+ processes
116
+ puts
117
+ queues
118
+ end
119
+
120
+ def version
121
+ puts "Sidekiq #{Sidekiq::VERSION}"
122
+ puts Time.now
123
+ end
124
+
125
+ def overview
126
+ puts '---- Overview ----'
127
+ puts " Processed: #{delimit stats.processed}"
128
+ puts " Failed: #{delimit stats.failed}"
129
+ puts " Busy: #{delimit stats.workers_size}"
130
+ puts " Enqueued: #{delimit stats.enqueued}"
131
+ puts " Retries: #{delimit stats.retry_size}"
132
+ puts " Scheduled: #{delimit stats.scheduled_size}"
133
+ puts " Dead: #{delimit stats.dead_size}"
134
+ end
135
+
136
+ def processes
137
+ puts "---- Processes (#{process_set.size}) ----"
138
+ process_set.each_with_index do |process, index|
139
+ puts "#{process['identity']} #{tags_for(process)}"
140
+ puts " Started: #{Time.at(process['started_at'])} (#{time_ago(process['started_at'])})"
141
+ puts " Threads: #{process['concurrency']} (#{process['busy']} busy)"
142
+ puts " Queues: #{split_multiline(process['queues'].sort, pad: 11)}"
143
+ puts '' unless (index+1) == process_set.size
144
+ end
145
+ end
146
+
147
+ COL_PAD = 2
148
+ def queues
149
+ puts "---- Queues (#{queue_data.size}) ----"
150
+ columns = {
151
+ name: [:ljust, (['name'] + queue_data.map(&:name)).map(&:length).max + COL_PAD],
152
+ size: [:rjust, (['size'] + queue_data.map(&:size)).map(&:length).max + COL_PAD],
153
+ latency: [:rjust, (['latency'] + queue_data.map(&:latency)).map(&:length).max + COL_PAD]
154
+ }
155
+ columns.each { |col, (dir, width)| print col.to_s.upcase.public_send(dir, width) }
156
+ puts
157
+ queue_data.each do |q|
158
+ columns.each do |col, (dir, width)|
159
+ print q.send(col).public_send(dir, width)
160
+ end
161
+ puts
162
+ end
163
+ end
164
+
165
+ private
166
+
167
+ def delimit(number)
168
+ number.to_s.reverse.scan(/.{1,3}/).join(',').reverse
169
+ end
170
+
171
+ def split_multiline(values, opts = {})
172
+ return 'none' unless values
173
+ pad = opts[:pad] || 0
174
+ max_length = opts[:max_length] || (80 - pad)
175
+ out = []
176
+ line = ''
177
+ values.each do |value|
178
+ if (line.length + value.length) > max_length
179
+ out << line
180
+ line = ' ' * pad
181
+ end
182
+ line << value + ', '
183
+ end
184
+ out << line[0..-3]
185
+ out.join("\n")
186
+ end
187
+
188
+ def tags_for(process)
189
+ tags = [
190
+ process['tag'],
191
+ process['labels'],
192
+ (process['quiet'] == 'true' ? 'quiet' : nil)
193
+ ].flatten.compact
194
+ tags.any? ? "[#{tags.join('] [')}]" : nil
195
+ end
196
+
197
+ def time_ago(timestamp)
198
+ seconds = Time.now - Time.at(timestamp)
199
+ return 'just now' if seconds < 60
200
+ return 'a minute ago' if seconds < 120
201
+ return "#{seconds.floor / 60} minutes ago" if seconds < 3600
202
+ return 'an hour ago' if seconds < 7200
203
+ "#{seconds.floor / 60 / 60} hours ago"
204
+ end
205
+
206
+ QUEUE_STRUCT = Struct.new(:name, :size, :latency)
207
+ def queue_data
208
+ @queue_data ||= Sidekiq::Queue.all.map do |q|
209
+ QUEUE_STRUCT.new(q.name, q.size.to_s, sprintf('%#.2f', q.latency))
210
+ end
211
+ end
212
+
213
+ def process_set
214
+ @process_set ||= Sidekiq::ProcessSet.new
215
+ end
216
+
217
+ def stats
218
+ @stats ||= Sidekiq::Stats.new
219
+ end
220
+ end
221
+ end
@@ -0,0 +1,42 @@
1
+ # frozen_string_literal: true
2
+ module Sidekiq
3
+ module Extensions
4
+
5
+ def self.enable_delay!
6
+ if defined?(::ActiveSupport)
7
+ require 'sidekiq/extensions/active_record'
8
+ require 'sidekiq/extensions/action_mailer'
9
+
10
+ # Need to patch Psych so it can autoload classes whose names are serialized
11
+ # in the delayed YAML.
12
+ Psych::Visitors::ToRuby.prepend(Sidekiq::Extensions::PsychAutoload)
13
+
14
+ ActiveSupport.on_load(:active_record) do
15
+ include Sidekiq::Extensions::ActiveRecord
16
+ end
17
+ ActiveSupport.on_load(:action_mailer) do
18
+ extend Sidekiq::Extensions::ActionMailer
19
+ end
20
+ end
21
+
22
+ require 'sidekiq/extensions/class_methods'
23
+ Module.__send__(:include, Sidekiq::Extensions::Klass)
24
+ end
25
+
26
+ module PsychAutoload
27
+ def resolve_class(klass_name)
28
+ return nil if !klass_name || klass_name.empty?
29
+ # constantize
30
+ names = klass_name.split('::')
31
+ names.shift if names.empty? || names.first.empty?
32
+
33
+ names.inject(Object) do |constant, name|
34
+ constant.const_defined?(name) ? constant.const_get(name) : constant.const_missing(name)
35
+ end
36
+ rescue NameError
37
+ super
38
+ end
39
+ end
40
+ end
41
+ end
42
+
@@ -0,0 +1,29 @@
1
+ # frozen_string_literal: true
2
+ require 'sidekiq'
3
+
4
+ module Sidekiq
5
+ module ExceptionHandler
6
+
7
+ class Logger
8
+ def call(ex, ctxHash)
9
+ Sidekiq.logger.warn(Sidekiq.dump_json(ctxHash)) if !ctxHash.empty?
10
+ Sidekiq.logger.warn("#{ex.class.name}: #{ex.message}")
11
+ Sidekiq.logger.warn(ex.backtrace.join("\n")) unless ex.backtrace.nil?
12
+ end
13
+
14
+ Sidekiq.error_handlers << Sidekiq::ExceptionHandler::Logger.new
15
+ end
16
+
17
+ def handle_exception(ex, ctxHash={})
18
+ Sidekiq.error_handlers.each do |handler|
19
+ begin
20
+ handler.call(ex, ctxHash)
21
+ rescue => ex
22
+ Sidekiq.logger.error "!!! ERROR HANDLER THREW AN ERROR !!!"
23
+ Sidekiq.logger.error ex
24
+ Sidekiq.logger.error ex.backtrace.join("\n") unless ex.backtrace.nil?
25
+ end
26
+ end
27
+ end
28
+ end
29
+ end
@@ -0,0 +1,57 @@
1
+ # frozen_string_literal: true
2
+ require 'sidekiq/extensions/generic_proxy'
3
+
4
+ module Sidekiq
5
+ module Extensions
6
+ ##
7
+ # Adds 'delay', 'delay_for' and `delay_until` methods to ActionMailer to offload arbitrary email
8
+ # delivery to Sidekiq. Example:
9
+ #
10
+ # UserMailer.delay.send_welcome_email(new_user)
11
+ # UserMailer.delay_for(5.days).send_welcome_email(new_user)
12
+ # UserMailer.delay_until(5.days.from_now).send_welcome_email(new_user)
13
+ class DelayedMailer
14
+ include Sidekiq::Worker
15
+
16
+ def perform(yml)
17
+ (target, method_name, args) = YAML.load(yml)
18
+ msg = target.public_send(method_name, *args)
19
+ # The email method can return nil, which causes ActionMailer to return
20
+ # an undeliverable empty message.
21
+ if msg
22
+ deliver(msg)
23
+ else
24
+ raise "#{target.name}##{method_name} returned an undeliverable mail object"
25
+ end
26
+ end
27
+
28
+ private
29
+
30
+ def deliver(msg)
31
+ if msg.respond_to?(:deliver_now)
32
+ # Rails 4.2/5.0
33
+ msg.deliver_now
34
+ else
35
+ # Rails 3.2/4.0/4.1
36
+ msg.deliver
37
+ end
38
+ end
39
+ end
40
+
41
+ module ActionMailer
42
+ def sidekiq_delay(options={})
43
+ Proxy.new(DelayedMailer, self, options)
44
+ end
45
+ def sidekiq_delay_for(interval, options={})
46
+ Proxy.new(DelayedMailer, self, options.merge('at' => Time.now.to_f + interval.to_f))
47
+ end
48
+ def sidekiq_delay_until(timestamp, options={})
49
+ Proxy.new(DelayedMailer, self, options.merge('at' => timestamp.to_f))
50
+ end
51
+ alias_method :delay, :sidekiq_delay
52
+ alias_method :delay_for, :sidekiq_delay_for
53
+ alias_method :delay_until, :sidekiq_delay_until
54
+ end
55
+
56
+ end
57
+ end
@@ -0,0 +1,40 @@
1
+ # frozen_string_literal: true
2
+ require 'sidekiq/extensions/generic_proxy'
3
+
4
+ module Sidekiq
5
+ module Extensions
6
+ ##
7
+ # Adds 'delay', 'delay_for' and `delay_until` methods to ActiveRecord to offload instance method
8
+ # execution to Sidekiq. Examples:
9
+ #
10
+ # User.recent_signups.each { |user| user.delay.mark_as_awesome }
11
+ #
12
+ # Please note, this is not recommended as this will serialize the entire
13
+ # object to Redis. Your Sidekiq jobs should pass IDs, not entire instances.
14
+ # This is here for backwards compatibility with Delayed::Job only.
15
+ class DelayedModel
16
+ include Sidekiq::Worker
17
+
18
+ def perform(yml)
19
+ (target, method_name, args) = YAML.load(yml)
20
+ target.__send__(method_name, *args)
21
+ end
22
+ end
23
+
24
+ module ActiveRecord
25
+ def sidekiq_delay(options={})
26
+ Proxy.new(DelayedModel, self, options)
27
+ end
28
+ def sidekiq_delay_for(interval, options={})
29
+ Proxy.new(DelayedModel, self, options.merge('at' => Time.now.to_f + interval.to_f))
30
+ end
31
+ def sidekiq_delay_until(timestamp, options={})
32
+ Proxy.new(DelayedModel, self, options.merge('at' => timestamp.to_f))
33
+ end
34
+ alias_method :delay, :sidekiq_delay
35
+ alias_method :delay_for, :sidekiq_delay_for
36
+ alias_method :delay_until, :sidekiq_delay_until
37
+ end
38
+
39
+ end
40
+ end
@@ -0,0 +1,40 @@
1
+ # frozen_string_literal: true
2
+ require 'sidekiq/extensions/generic_proxy'
3
+
4
+ module Sidekiq
5
+ module Extensions
6
+ ##
7
+ # Adds 'delay', 'delay_for' and `delay_until` methods to all Classes to offload class method
8
+ # execution to Sidekiq. Examples:
9
+ #
10
+ # User.delay.delete_inactive
11
+ # Wikipedia.delay.download_changes_for(Date.today)
12
+ #
13
+ class DelayedClass
14
+ include Sidekiq::Worker
15
+
16
+ def perform(yml)
17
+ (target, method_name, args) = YAML.load(yml)
18
+ target.__send__(method_name, *args)
19
+ end
20
+ end
21
+
22
+ module Klass
23
+ def sidekiq_delay(options={})
24
+ Proxy.new(DelayedClass, self, options)
25
+ end
26
+ def sidekiq_delay_for(interval, options={})
27
+ Proxy.new(DelayedClass, self, options.merge('at' => Time.now.to_f + interval.to_f))
28
+ end
29
+ def sidekiq_delay_until(timestamp, options={})
30
+ Proxy.new(DelayedClass, self, options.merge('at' => timestamp.to_f))
31
+ end
32
+ alias_method :delay, :sidekiq_delay
33
+ alias_method :delay_for, :sidekiq_delay_for
34
+ alias_method :delay_until, :sidekiq_delay_until
35
+ end
36
+
37
+ end
38
+ end
39
+
40
+ Module.__send__(:include, Sidekiq::Extensions::Klass) unless defined?(::Rails)
@@ -0,0 +1,31 @@
1
+ # frozen_string_literal: true
2
+ require 'yaml'
3
+
4
+ module Sidekiq
5
+ module Extensions
6
+ SIZE_LIMIT = 8_192
7
+
8
+ class Proxy < BasicObject
9
+ def initialize(performable, target, options={})
10
+ @performable = performable
11
+ @target = target
12
+ @opts = options
13
+ end
14
+
15
+ def method_missing(name, *args)
16
+ # Sidekiq has a limitation in that its message must be JSON.
17
+ # JSON can't round trip real Ruby objects so we use YAML to
18
+ # serialize the objects to a String. The YAML will be converted
19
+ # to JSON and then deserialized on the other side back into a
20
+ # Ruby object.
21
+ obj = [@target, name, args]
22
+ marshalled = ::YAML.dump(obj)
23
+ if marshalled.size > SIZE_LIMIT
24
+ ::Sidekiq.logger.warn { "#{@target}.#{name} job argument is #{marshalled.bytesize} bytes, you should refactor it to reduce the size" }
25
+ end
26
+ @performable.client_push({ 'class' => @performable, 'args' => [marshalled] }.merge(@opts))
27
+ end
28
+ end
29
+
30
+ end
31
+ end