sidekiq 5.2.6 → 7.1.0

Sign up to get free protection for your applications and to get access to all the features.

Potentially problematic release.


This version of sidekiq might be problematic. Click here for more details.

Files changed (148) hide show
  1. checksums.yaml +4 -4
  2. data/Changes.md +537 -8
  3. data/LICENSE.txt +9 -0
  4. data/README.md +47 -50
  5. data/bin/sidekiq +22 -3
  6. data/bin/sidekiqload +213 -115
  7. data/bin/sidekiqmon +11 -0
  8. data/lib/generators/sidekiq/job_generator.rb +57 -0
  9. data/lib/generators/sidekiq/templates/{worker.rb.erb → job.rb.erb} +2 -2
  10. data/lib/generators/sidekiq/templates/{worker_spec.rb.erb → job_spec.rb.erb} +1 -1
  11. data/lib/generators/sidekiq/templates/{worker_test.rb.erb → job_test.rb.erb} +1 -1
  12. data/lib/sidekiq/api.rb +556 -351
  13. data/lib/sidekiq/capsule.rb +127 -0
  14. data/lib/sidekiq/cli.rb +203 -226
  15. data/lib/sidekiq/client.rb +121 -101
  16. data/lib/sidekiq/component.rb +68 -0
  17. data/lib/sidekiq/config.rb +274 -0
  18. data/lib/sidekiq/deploy.rb +62 -0
  19. data/lib/sidekiq/embedded.rb +61 -0
  20. data/lib/sidekiq/fetch.rb +49 -42
  21. data/lib/sidekiq/job.rb +374 -0
  22. data/lib/sidekiq/job_logger.rb +33 -7
  23. data/lib/sidekiq/job_retry.rb +131 -108
  24. data/lib/sidekiq/job_util.rb +105 -0
  25. data/lib/sidekiq/launcher.rb +203 -105
  26. data/lib/sidekiq/logger.rb +131 -0
  27. data/lib/sidekiq/manager.rb +43 -46
  28. data/lib/sidekiq/metrics/query.rb +153 -0
  29. data/lib/sidekiq/metrics/shared.rb +95 -0
  30. data/lib/sidekiq/metrics/tracking.rb +136 -0
  31. data/lib/sidekiq/middleware/chain.rb +113 -56
  32. data/lib/sidekiq/middleware/current_attributes.rb +56 -0
  33. data/lib/sidekiq/middleware/i18n.rb +7 -7
  34. data/lib/sidekiq/middleware/modules.rb +21 -0
  35. data/lib/sidekiq/monitor.rb +146 -0
  36. data/lib/sidekiq/paginator.rb +28 -16
  37. data/lib/sidekiq/processor.rb +108 -107
  38. data/lib/sidekiq/rails.rb +49 -38
  39. data/lib/sidekiq/redis_client_adapter.rb +96 -0
  40. data/lib/sidekiq/redis_connection.rb +38 -107
  41. data/lib/sidekiq/ring_buffer.rb +29 -0
  42. data/lib/sidekiq/scheduled.rb +111 -49
  43. data/lib/sidekiq/sd_notify.rb +149 -0
  44. data/lib/sidekiq/systemd.rb +24 -0
  45. data/lib/sidekiq/testing/inline.rb +6 -5
  46. data/lib/sidekiq/testing.rb +66 -84
  47. data/lib/sidekiq/transaction_aware_client.rb +44 -0
  48. data/lib/sidekiq/version.rb +3 -1
  49. data/lib/sidekiq/web/action.rb +15 -11
  50. data/lib/sidekiq/web/application.rb +123 -79
  51. data/lib/sidekiq/web/csrf_protection.rb +180 -0
  52. data/lib/sidekiq/web/helpers.rb +137 -106
  53. data/lib/sidekiq/web/router.rb +23 -19
  54. data/lib/sidekiq/web.rb +56 -107
  55. data/lib/sidekiq/worker_compatibility_alias.rb +13 -0
  56. data/lib/sidekiq.rb +92 -182
  57. data/sidekiq.gemspec +25 -16
  58. data/web/assets/images/apple-touch-icon.png +0 -0
  59. data/web/assets/javascripts/application.js +130 -61
  60. data/web/assets/javascripts/base-charts.js +106 -0
  61. data/web/assets/javascripts/chart.min.js +13 -0
  62. data/web/assets/javascripts/chartjs-plugin-annotation.min.js +7 -0
  63. data/web/assets/javascripts/dashboard-charts.js +166 -0
  64. data/web/assets/javascripts/dashboard.js +36 -292
  65. data/web/assets/javascripts/metrics.js +264 -0
  66. data/web/assets/stylesheets/application-dark.css +147 -0
  67. data/web/assets/stylesheets/application-rtl.css +2 -95
  68. data/web/assets/stylesheets/application.css +102 -522
  69. data/web/locales/ar.yml +71 -65
  70. data/web/locales/cs.yml +62 -62
  71. data/web/locales/da.yml +60 -53
  72. data/web/locales/de.yml +65 -53
  73. data/web/locales/el.yml +43 -24
  74. data/web/locales/en.yml +84 -66
  75. data/web/locales/es.yml +70 -54
  76. data/web/locales/fa.yml +65 -65
  77. data/web/locales/fr.yml +83 -62
  78. data/web/locales/gd.yml +99 -0
  79. data/web/locales/he.yml +65 -64
  80. data/web/locales/hi.yml +59 -59
  81. data/web/locales/it.yml +53 -53
  82. data/web/locales/ja.yml +75 -64
  83. data/web/locales/ko.yml +52 -52
  84. data/web/locales/lt.yml +83 -0
  85. data/web/locales/nb.yml +61 -61
  86. data/web/locales/nl.yml +52 -52
  87. data/web/locales/pl.yml +45 -45
  88. data/web/locales/pt-br.yml +63 -55
  89. data/web/locales/pt.yml +51 -51
  90. data/web/locales/ru.yml +68 -63
  91. data/web/locales/sv.yml +53 -53
  92. data/web/locales/ta.yml +60 -60
  93. data/web/locales/uk.yml +62 -61
  94. data/web/locales/ur.yml +64 -64
  95. data/web/locales/vi.yml +83 -0
  96. data/web/locales/zh-cn.yml +43 -16
  97. data/web/locales/zh-tw.yml +42 -8
  98. data/web/views/_footer.erb +6 -3
  99. data/web/views/_job_info.erb +21 -4
  100. data/web/views/_metrics_period_select.erb +12 -0
  101. data/web/views/_nav.erb +1 -1
  102. data/web/views/_paging.erb +2 -0
  103. data/web/views/_poll_link.erb +3 -6
  104. data/web/views/_summary.erb +7 -7
  105. data/web/views/busy.erb +75 -25
  106. data/web/views/dashboard.erb +58 -18
  107. data/web/views/dead.erb +3 -3
  108. data/web/views/layout.erb +3 -1
  109. data/web/views/metrics.erb +82 -0
  110. data/web/views/metrics_for_job.erb +68 -0
  111. data/web/views/morgue.erb +14 -15
  112. data/web/views/queue.erb +33 -24
  113. data/web/views/queues.erb +13 -3
  114. data/web/views/retries.erb +16 -17
  115. data/web/views/retry.erb +3 -3
  116. data/web/views/scheduled.erb +17 -15
  117. metadata +69 -69
  118. data/.github/contributing.md +0 -32
  119. data/.github/issue_template.md +0 -11
  120. data/.gitignore +0 -15
  121. data/.travis.yml +0 -11
  122. data/3.0-Upgrade.md +0 -70
  123. data/4.0-Upgrade.md +0 -53
  124. data/5.0-Upgrade.md +0 -56
  125. data/COMM-LICENSE +0 -97
  126. data/Ent-Changes.md +0 -238
  127. data/Gemfile +0 -23
  128. data/LICENSE +0 -9
  129. data/Pro-2.0-Upgrade.md +0 -138
  130. data/Pro-3.0-Upgrade.md +0 -44
  131. data/Pro-4.0-Upgrade.md +0 -35
  132. data/Pro-Changes.md +0 -759
  133. data/Rakefile +0 -9
  134. data/bin/sidekiqctl +0 -20
  135. data/code_of_conduct.md +0 -50
  136. data/lib/generators/sidekiq/worker_generator.rb +0 -49
  137. data/lib/sidekiq/core_ext.rb +0 -1
  138. data/lib/sidekiq/ctl.rb +0 -221
  139. data/lib/sidekiq/delay.rb +0 -42
  140. data/lib/sidekiq/exception_handler.rb +0 -29
  141. data/lib/sidekiq/extensions/action_mailer.rb +0 -57
  142. data/lib/sidekiq/extensions/active_record.rb +0 -40
  143. data/lib/sidekiq/extensions/class_methods.rb +0 -40
  144. data/lib/sidekiq/extensions/generic_proxy.rb +0 -31
  145. data/lib/sidekiq/logging.rb +0 -122
  146. data/lib/sidekiq/middleware/server/active_record.rb +0 -23
  147. data/lib/sidekiq/util.rb +0 -66
  148. data/lib/sidekiq/worker.rb +0 -220
@@ -1,122 +0,0 @@
1
- # frozen_string_literal: true
2
- require 'time'
3
- require 'logger'
4
- require 'fcntl'
5
-
6
- module Sidekiq
7
- module Logging
8
-
9
- class Pretty < Logger::Formatter
10
- SPACE = " "
11
-
12
- # Provide a call() method that returns the formatted message.
13
- def call(severity, time, program_name, message)
14
- "#{time.utc.iso8601(3)} #{::Process.pid} TID-#{Sidekiq::Logging.tid}#{context} #{severity}: #{message}\n"
15
- end
16
-
17
- def context
18
- c = Thread.current[:sidekiq_context]
19
- " #{c.join(SPACE)}" if c && c.any?
20
- end
21
- end
22
-
23
- class WithoutTimestamp < Pretty
24
- def call(severity, time, program_name, message)
25
- "#{::Process.pid} TID-#{Sidekiq::Logging.tid}#{context} #{severity}: #{message}\n"
26
- end
27
- end
28
-
29
- def self.tid
30
- Thread.current['sidekiq_tid'] ||= (Thread.current.object_id ^ ::Process.pid).to_s(36)
31
- end
32
-
33
- def self.job_hash_context(job_hash)
34
- # If we're using a wrapper class, like ActiveJob, use the "wrapped"
35
- # attribute to expose the underlying thing.
36
- klass = job_hash['wrapped'] || job_hash["class"]
37
- bid = job_hash['bid']
38
- "#{klass} JID-#{job_hash['jid']}#{" BID-#{bid}" if bid}"
39
- end
40
-
41
- def self.with_job_hash_context(job_hash, &block)
42
- with_context(job_hash_context(job_hash), &block)
43
- end
44
-
45
- def self.with_context(msg)
46
- Thread.current[:sidekiq_context] ||= []
47
- Thread.current[:sidekiq_context] << msg
48
- yield
49
- ensure
50
- Thread.current[:sidekiq_context].pop
51
- end
52
-
53
- def self.initialize_logger(log_target = STDOUT)
54
- oldlogger = defined?(@logger) ? @logger : nil
55
- @logger = Logger.new(log_target)
56
- @logger.level = Logger::INFO
57
- @logger.formatter = ENV['DYNO'] ? WithoutTimestamp.new : Pretty.new
58
- oldlogger.close if oldlogger && !$TESTING # don't want to close testing's STDOUT logging
59
- @logger
60
- end
61
-
62
- def self.logger
63
- defined?(@logger) ? @logger : initialize_logger
64
- end
65
-
66
- def self.logger=(log)
67
- @logger = (log ? log : Logger.new(File::NULL))
68
- end
69
-
70
- # This reopens ALL logfiles in the process that have been rotated
71
- # using logrotate(8) (without copytruncate) or similar tools.
72
- # A +File+ object is considered for reopening if it is:
73
- # 1) opened with the O_APPEND and O_WRONLY flags
74
- # 2) the current open file handle does not match its original open path
75
- # 3) unbuffered (as far as userspace buffering goes, not O_SYNC)
76
- # Returns the number of files reopened
77
- def self.reopen_logs
78
- to_reopen = []
79
- append_flags = File::WRONLY | File::APPEND
80
-
81
- ObjectSpace.each_object(File) do |fp|
82
- begin
83
- if !fp.closed? && fp.stat.file? && fp.sync && (fp.fcntl(Fcntl::F_GETFL) & append_flags) == append_flags
84
- to_reopen << fp
85
- end
86
- rescue IOError, Errno::EBADF
87
- end
88
- end
89
-
90
- nr = 0
91
- to_reopen.each do |fp|
92
- orig_st = begin
93
- fp.stat
94
- rescue IOError, Errno::EBADF
95
- next
96
- end
97
-
98
- begin
99
- b = File.stat(fp.path)
100
- next if orig_st.ino == b.ino && orig_st.dev == b.dev
101
- rescue Errno::ENOENT
102
- end
103
-
104
- begin
105
- File.open(fp.path, 'a') { |tmpfp| fp.reopen(tmpfp) }
106
- fp.sync = true
107
- nr += 1
108
- rescue IOError, Errno::EBADF
109
- # not much we can do...
110
- end
111
- end
112
- nr
113
- rescue RuntimeError => ex
114
- # RuntimeError: ObjectSpace is disabled; each_object will only work with Class, pass -X+O to enable
115
- puts "Unable to reopen logs: #{ex.message}"
116
- end
117
-
118
- def logger
119
- Sidekiq::Logging.logger
120
- end
121
- end
122
- end
@@ -1,23 +0,0 @@
1
- # frozen_string_literal: true
2
- module Sidekiq
3
- module Middleware
4
- module Server
5
- class ActiveRecord
6
-
7
- def initialize
8
- # With Rails 5+ we must use the Reloader **always**.
9
- # The reloader handles code loading and db connection management.
10
- if defined?(::Rails) && ::Rails::VERSION::MAJOR >= 5
11
- raise ArgumentError, "Rails 5 no longer needs or uses the ActiveRecord middleware."
12
- end
13
- end
14
-
15
- def call(*args)
16
- yield
17
- ensure
18
- ::ActiveRecord::Base.clear_active_connections!
19
- end
20
- end
21
- end
22
- end
23
- end
data/lib/sidekiq/util.rb DELETED
@@ -1,66 +0,0 @@
1
- # frozen_string_literal: true
2
- require 'socket'
3
- require 'securerandom'
4
- require 'sidekiq/exception_handler'
5
-
6
- module Sidekiq
7
- ##
8
- # This module is part of Sidekiq core and not intended for extensions.
9
- #
10
- module Util
11
- include ExceptionHandler
12
-
13
- EXPIRY = 60 * 60 * 24
14
-
15
- def watchdog(last_words)
16
- yield
17
- rescue Exception => ex
18
- handle_exception(ex, { context: last_words })
19
- raise ex
20
- end
21
-
22
- def safe_thread(name, &block)
23
- Thread.new do
24
- Thread.current['sidekiq_label'] = name
25
- watchdog(name, &block)
26
- end
27
- end
28
-
29
- def logger
30
- Sidekiq.logger
31
- end
32
-
33
- def redis(&block)
34
- Sidekiq.redis(&block)
35
- end
36
-
37
- def hostname
38
- ENV['DYNO'] || Socket.gethostname
39
- end
40
-
41
- def process_nonce
42
- @@process_nonce ||= SecureRandom.hex(6)
43
- end
44
-
45
- def identity
46
- @@identity ||= "#{hostname}:#{$$}:#{process_nonce}"
47
- end
48
-
49
- def fire_event(event, options={})
50
- reverse = options[:reverse]
51
- reraise = options[:reraise]
52
-
53
- arr = Sidekiq.options[:lifecycle_events][event]
54
- arr.reverse! if reverse
55
- arr.each do |block|
56
- begin
57
- block.call
58
- rescue => ex
59
- handle_exception(ex, { context: "Exception during Sidekiq lifecycle event.", event: event })
60
- raise ex if reraise
61
- end
62
- end
63
- arr.clear
64
- end
65
- end
66
- end
@@ -1,220 +0,0 @@
1
- # frozen_string_literal: true
2
- require 'sidekiq/client'
3
-
4
- module Sidekiq
5
-
6
- ##
7
- # Include this module in your worker class and you can easily create
8
- # asynchronous jobs:
9
- #
10
- # class HardWorker
11
- # include Sidekiq::Worker
12
- #
13
- # def perform(*args)
14
- # # do some work
15
- # end
16
- # end
17
- #
18
- # Then in your Rails app, you can do this:
19
- #
20
- # HardWorker.perform_async(1, 2, 3)
21
- #
22
- # Note that perform_async is a class method, perform is an instance method.
23
- module Worker
24
- attr_accessor :jid
25
-
26
- def self.included(base)
27
- raise ArgumentError, "You cannot include Sidekiq::Worker in an ActiveJob: #{base.name}" if base.ancestors.any? {|c| c.name == 'ActiveJob::Base' }
28
-
29
- base.extend(ClassMethods)
30
- base.sidekiq_class_attribute :sidekiq_options_hash
31
- base.sidekiq_class_attribute :sidekiq_retry_in_block
32
- base.sidekiq_class_attribute :sidekiq_retries_exhausted_block
33
- end
34
-
35
- def logger
36
- Sidekiq.logger
37
- end
38
-
39
- # This helper class encapsulates the set options for `set`, e.g.
40
- #
41
- # SomeWorker.set(queue: 'foo').perform_async(....)
42
- #
43
- class Setter
44
- def initialize(klass, opts)
45
- @klass = klass
46
- @opts = opts
47
- end
48
-
49
- def set(options)
50
- @opts.merge!(options)
51
- self
52
- end
53
-
54
- def perform_async(*args)
55
- @klass.client_push(@opts.merge('args' => args, 'class' => @klass))
56
- end
57
-
58
- # +interval+ must be a timestamp, numeric or something that acts
59
- # numeric (like an activesupport time interval).
60
- def perform_in(interval, *args)
61
- int = interval.to_f
62
- now = Time.now.to_f
63
- ts = (int < 1_000_000_000 ? now + int : int)
64
-
65
- payload = @opts.merge('class' => @klass, 'args' => args, 'at' => ts)
66
- # Optimization to enqueue something now that is scheduled to go out now or in the past
67
- payload.delete('at') if ts <= now
68
- @klass.client_push(payload)
69
- end
70
- alias_method :perform_at, :perform_in
71
- end
72
-
73
- module ClassMethods
74
- ACCESSOR_MUTEX = Mutex.new
75
-
76
- def delay(*args)
77
- raise ArgumentError, "Do not call .delay on a Sidekiq::Worker class, call .perform_async"
78
- end
79
-
80
- def delay_for(*args)
81
- raise ArgumentError, "Do not call .delay_for on a Sidekiq::Worker class, call .perform_in"
82
- end
83
-
84
- def delay_until(*args)
85
- raise ArgumentError, "Do not call .delay_until on a Sidekiq::Worker class, call .perform_at"
86
- end
87
-
88
- def set(options)
89
- Setter.new(self, options)
90
- end
91
-
92
- def perform_async(*args)
93
- client_push('class' => self, 'args' => args)
94
- end
95
-
96
- # +interval+ must be a timestamp, numeric or something that acts
97
- # numeric (like an activesupport time interval).
98
- def perform_in(interval, *args)
99
- int = interval.to_f
100
- now = Time.now.to_f
101
- ts = (int < 1_000_000_000 ? now + int : int)
102
-
103
- item = { 'class' => self, 'args' => args, 'at' => ts }
104
-
105
- # Optimization to enqueue something now that is scheduled to go out now or in the past
106
- item.delete('at') if ts <= now
107
-
108
- client_push(item)
109
- end
110
- alias_method :perform_at, :perform_in
111
-
112
- ##
113
- # Allows customization for this type of Worker.
114
- # Legal options:
115
- #
116
- # queue - use a named queue for this Worker, default 'default'
117
- # retry - enable the RetryJobs middleware for this Worker, *true* to use the default
118
- # or *Integer* count
119
- # backtrace - whether to save any error backtrace in the retry payload to display in web UI,
120
- # can be true, false or an integer number of lines to save, default *false*
121
- # pool - use the given Redis connection pool to push this type of job to a given shard.
122
- #
123
- # In practice, any option is allowed. This is the main mechanism to configure the
124
- # options for a specific job.
125
- def sidekiq_options(opts={})
126
- # stringify
127
- self.sidekiq_options_hash = get_sidekiq_options.merge(Hash[opts.map{|k, v| [k.to_s, v]}])
128
- end
129
-
130
- def sidekiq_retry_in(&block)
131
- self.sidekiq_retry_in_block = block
132
- end
133
-
134
- def sidekiq_retries_exhausted(&block)
135
- self.sidekiq_retries_exhausted_block = block
136
- end
137
-
138
- def get_sidekiq_options # :nodoc:
139
- self.sidekiq_options_hash ||= Sidekiq.default_worker_options
140
- end
141
-
142
- def client_push(item) # :nodoc:
143
- pool = Thread.current[:sidekiq_via_pool] || get_sidekiq_options['pool'] || Sidekiq.redis_pool
144
- # stringify
145
- item.keys.each do |key|
146
- item[key.to_s] = item.delete(key)
147
- end
148
-
149
- Sidekiq::Client.new(pool).push(item)
150
- end
151
-
152
- def sidekiq_class_attribute(*attrs)
153
- instance_reader = true
154
- instance_writer = true
155
-
156
- attrs.each do |name|
157
- synchronized_getter = "__synchronized_#{name}"
158
-
159
- singleton_class.instance_eval do
160
- undef_method(name) if method_defined?(name) || private_method_defined?(name)
161
- end
162
-
163
- define_singleton_method(synchronized_getter) { nil }
164
- singleton_class.class_eval do
165
- private(synchronized_getter)
166
- end
167
-
168
- define_singleton_method(name) { ACCESSOR_MUTEX.synchronize { send synchronized_getter } }
169
-
170
- ivar = "@#{name}"
171
-
172
- singleton_class.instance_eval do
173
- m = "#{name}="
174
- undef_method(m) if method_defined?(m) || private_method_defined?(m)
175
- end
176
- define_singleton_method("#{name}=") do |val|
177
- singleton_class.class_eval do
178
- ACCESSOR_MUTEX.synchronize do
179
- undef_method(synchronized_getter) if method_defined?(synchronized_getter) || private_method_defined?(synchronized_getter)
180
- define_method(synchronized_getter) { val }
181
- end
182
- end
183
-
184
- if singleton_class?
185
- class_eval do
186
- undef_method(name) if method_defined?(name) || private_method_defined?(name)
187
- define_method(name) do
188
- if instance_variable_defined? ivar
189
- instance_variable_get ivar
190
- else
191
- singleton_class.send name
192
- end
193
- end
194
- end
195
- end
196
- val
197
- end
198
-
199
- if instance_reader
200
- undef_method(name) if method_defined?(name) || private_method_defined?(name)
201
- define_method(name) do
202
- if instance_variable_defined?(ivar)
203
- instance_variable_get ivar
204
- else
205
- self.class.public_send name
206
- end
207
- end
208
- end
209
-
210
- if instance_writer
211
- m = "#{name}="
212
- undef_method(m) if method_defined?(m) || private_method_defined?(m)
213
- attr_writer name
214
- end
215
- end
216
- end
217
-
218
- end
219
- end
220
- end