sidekiq 5.2.4 → 7.2.4

Sign up to get free protection for your applications and to get access to all the features.
Files changed (153) hide show
  1. checksums.yaml +4 -4
  2. data/Changes.md +672 -8
  3. data/LICENSE.txt +9 -0
  4. data/README.md +48 -51
  5. data/bin/multi_queue_bench +271 -0
  6. data/bin/sidekiq +22 -3
  7. data/bin/sidekiqload +213 -115
  8. data/bin/sidekiqmon +11 -0
  9. data/lib/generators/sidekiq/job_generator.rb +57 -0
  10. data/lib/generators/sidekiq/templates/{worker.rb.erb → job.rb.erb} +2 -2
  11. data/lib/generators/sidekiq/templates/{worker_spec.rb.erb → job_spec.rb.erb} +1 -1
  12. data/lib/generators/sidekiq/templates/{worker_test.rb.erb → job_test.rb.erb} +1 -1
  13. data/lib/sidekiq/api.rb +623 -352
  14. data/lib/sidekiq/capsule.rb +127 -0
  15. data/lib/sidekiq/cli.rb +214 -229
  16. data/lib/sidekiq/client.rb +127 -102
  17. data/lib/sidekiq/component.rb +68 -0
  18. data/lib/sidekiq/config.rb +287 -0
  19. data/lib/sidekiq/deploy.rb +62 -0
  20. data/lib/sidekiq/embedded.rb +61 -0
  21. data/lib/sidekiq/fetch.rb +49 -42
  22. data/lib/sidekiq/job.rb +374 -0
  23. data/lib/sidekiq/job_logger.rb +33 -7
  24. data/lib/sidekiq/job_retry.rb +157 -108
  25. data/lib/sidekiq/job_util.rb +107 -0
  26. data/lib/sidekiq/launcher.rb +206 -106
  27. data/lib/sidekiq/logger.rb +131 -0
  28. data/lib/sidekiq/manager.rb +43 -46
  29. data/lib/sidekiq/metrics/query.rb +156 -0
  30. data/lib/sidekiq/metrics/shared.rb +95 -0
  31. data/lib/sidekiq/metrics/tracking.rb +140 -0
  32. data/lib/sidekiq/middleware/chain.rb +113 -56
  33. data/lib/sidekiq/middleware/current_attributes.rb +95 -0
  34. data/lib/sidekiq/middleware/i18n.rb +7 -7
  35. data/lib/sidekiq/middleware/modules.rb +21 -0
  36. data/lib/sidekiq/monitor.rb +146 -0
  37. data/lib/sidekiq/paginator.rb +28 -16
  38. data/lib/sidekiq/processor.rb +126 -117
  39. data/lib/sidekiq/rails.rb +52 -38
  40. data/lib/sidekiq/redis_client_adapter.rb +111 -0
  41. data/lib/sidekiq/redis_connection.rb +41 -112
  42. data/lib/sidekiq/ring_buffer.rb +29 -0
  43. data/lib/sidekiq/scheduled.rb +112 -50
  44. data/lib/sidekiq/sd_notify.rb +149 -0
  45. data/lib/sidekiq/systemd.rb +24 -0
  46. data/lib/sidekiq/testing/inline.rb +6 -5
  47. data/lib/sidekiq/testing.rb +91 -90
  48. data/lib/sidekiq/transaction_aware_client.rb +51 -0
  49. data/lib/sidekiq/version.rb +3 -1
  50. data/lib/sidekiq/web/action.rb +20 -11
  51. data/lib/sidekiq/web/application.rb +202 -80
  52. data/lib/sidekiq/web/csrf_protection.rb +183 -0
  53. data/lib/sidekiq/web/helpers.rb +165 -114
  54. data/lib/sidekiq/web/router.rb +23 -19
  55. data/lib/sidekiq/web.rb +68 -107
  56. data/lib/sidekiq/worker_compatibility_alias.rb +13 -0
  57. data/lib/sidekiq.rb +92 -182
  58. data/sidekiq.gemspec +25 -16
  59. data/web/assets/images/apple-touch-icon.png +0 -0
  60. data/web/assets/javascripts/application.js +152 -61
  61. data/web/assets/javascripts/base-charts.js +106 -0
  62. data/web/assets/javascripts/chart.min.js +13 -0
  63. data/web/assets/javascripts/chartjs-plugin-annotation.min.js +7 -0
  64. data/web/assets/javascripts/dashboard-charts.js +182 -0
  65. data/web/assets/javascripts/dashboard.js +35 -293
  66. data/web/assets/javascripts/metrics.js +298 -0
  67. data/web/assets/stylesheets/application-dark.css +147 -0
  68. data/web/assets/stylesheets/application-rtl.css +10 -93
  69. data/web/assets/stylesheets/application.css +124 -522
  70. data/web/assets/stylesheets/bootstrap.css +1 -1
  71. data/web/locales/ar.yml +71 -65
  72. data/web/locales/cs.yml +62 -62
  73. data/web/locales/da.yml +60 -53
  74. data/web/locales/de.yml +65 -53
  75. data/web/locales/el.yml +43 -24
  76. data/web/locales/en.yml +86 -66
  77. data/web/locales/es.yml +70 -54
  78. data/web/locales/fa.yml +65 -65
  79. data/web/locales/fr.yml +83 -62
  80. data/web/locales/gd.yml +99 -0
  81. data/web/locales/he.yml +65 -64
  82. data/web/locales/hi.yml +59 -59
  83. data/web/locales/it.yml +53 -53
  84. data/web/locales/ja.yml +75 -64
  85. data/web/locales/ko.yml +52 -52
  86. data/web/locales/lt.yml +83 -0
  87. data/web/locales/nb.yml +61 -61
  88. data/web/locales/nl.yml +52 -52
  89. data/web/locales/pl.yml +45 -45
  90. data/web/locales/pt-br.yml +83 -55
  91. data/web/locales/pt.yml +51 -51
  92. data/web/locales/ru.yml +68 -63
  93. data/web/locales/sv.yml +53 -53
  94. data/web/locales/ta.yml +60 -60
  95. data/web/locales/uk.yml +62 -61
  96. data/web/locales/ur.yml +64 -64
  97. data/web/locales/vi.yml +83 -0
  98. data/web/locales/zh-cn.yml +43 -16
  99. data/web/locales/zh-tw.yml +42 -8
  100. data/web/views/_footer.erb +18 -3
  101. data/web/views/_job_info.erb +21 -4
  102. data/web/views/_metrics_period_select.erb +12 -0
  103. data/web/views/_nav.erb +1 -1
  104. data/web/views/_paging.erb +2 -0
  105. data/web/views/_poll_link.erb +3 -6
  106. data/web/views/_summary.erb +7 -7
  107. data/web/views/busy.erb +79 -29
  108. data/web/views/dashboard.erb +48 -18
  109. data/web/views/dead.erb +3 -3
  110. data/web/views/filtering.erb +7 -0
  111. data/web/views/layout.erb +3 -1
  112. data/web/views/metrics.erb +91 -0
  113. data/web/views/metrics_for_job.erb +59 -0
  114. data/web/views/morgue.erb +14 -15
  115. data/web/views/queue.erb +33 -24
  116. data/web/views/queues.erb +19 -5
  117. data/web/views/retries.erb +16 -17
  118. data/web/views/retry.erb +3 -3
  119. data/web/views/scheduled.erb +17 -15
  120. metadata +71 -72
  121. data/.github/contributing.md +0 -32
  122. data/.github/issue_template.md +0 -11
  123. data/.gitignore +0 -15
  124. data/.travis.yml +0 -17
  125. data/3.0-Upgrade.md +0 -70
  126. data/4.0-Upgrade.md +0 -53
  127. data/5.0-Upgrade.md +0 -56
  128. data/Appraisals +0 -9
  129. data/COMM-LICENSE +0 -95
  130. data/Ent-Changes.md +0 -225
  131. data/Gemfile +0 -29
  132. data/LICENSE +0 -9
  133. data/Pro-2.0-Upgrade.md +0 -138
  134. data/Pro-3.0-Upgrade.md +0 -44
  135. data/Pro-4.0-Upgrade.md +0 -35
  136. data/Pro-Changes.md +0 -752
  137. data/Rakefile +0 -9
  138. data/bin/sidekiqctl +0 -237
  139. data/code_of_conduct.md +0 -50
  140. data/gemfiles/rails_4.gemfile +0 -31
  141. data/gemfiles/rails_5.gemfile +0 -31
  142. data/lib/generators/sidekiq/worker_generator.rb +0 -49
  143. data/lib/sidekiq/core_ext.rb +0 -1
  144. data/lib/sidekiq/delay.rb +0 -42
  145. data/lib/sidekiq/exception_handler.rb +0 -29
  146. data/lib/sidekiq/extensions/action_mailer.rb +0 -57
  147. data/lib/sidekiq/extensions/active_record.rb +0 -40
  148. data/lib/sidekiq/extensions/class_methods.rb +0 -40
  149. data/lib/sidekiq/extensions/generic_proxy.rb +0 -31
  150. data/lib/sidekiq/logging.rb +0 -122
  151. data/lib/sidekiq/middleware/server/active_record.rb +0 -23
  152. data/lib/sidekiq/util.rb +0 -66
  153. data/lib/sidekiq/worker.rb +0 -215
@@ -1,122 +0,0 @@
1
- # frozen_string_literal: true
2
- require 'time'
3
- require 'logger'
4
- require 'fcntl'
5
-
6
- module Sidekiq
7
- module Logging
8
-
9
- class Pretty < Logger::Formatter
10
- SPACE = " "
11
-
12
- # Provide a call() method that returns the formatted message.
13
- def call(severity, time, program_name, message)
14
- "#{time.utc.iso8601(3)} #{::Process.pid} TID-#{Sidekiq::Logging.tid}#{context} #{severity}: #{message}\n"
15
- end
16
-
17
- def context
18
- c = Thread.current[:sidekiq_context]
19
- " #{c.join(SPACE)}" if c && c.any?
20
- end
21
- end
22
-
23
- class WithoutTimestamp < Pretty
24
- def call(severity, time, program_name, message)
25
- "#{::Process.pid} TID-#{Sidekiq::Logging.tid}#{context} #{severity}: #{message}\n"
26
- end
27
- end
28
-
29
- def self.tid
30
- Thread.current['sidekiq_tid'] ||= (Thread.current.object_id ^ ::Process.pid).to_s(36)
31
- end
32
-
33
- def self.job_hash_context(job_hash)
34
- # If we're using a wrapper class, like ActiveJob, use the "wrapped"
35
- # attribute to expose the underlying thing.
36
- klass = job_hash['wrapped'] || job_hash["class"]
37
- bid = job_hash['bid']
38
- "#{klass} JID-#{job_hash['jid']}#{" BID-#{bid}" if bid}"
39
- end
40
-
41
- def self.with_job_hash_context(job_hash, &block)
42
- with_context(job_hash_context(job_hash), &block)
43
- end
44
-
45
- def self.with_context(msg)
46
- Thread.current[:sidekiq_context] ||= []
47
- Thread.current[:sidekiq_context] << msg
48
- yield
49
- ensure
50
- Thread.current[:sidekiq_context].pop
51
- end
52
-
53
- def self.initialize_logger(log_target = STDOUT)
54
- oldlogger = defined?(@logger) ? @logger : nil
55
- @logger = Logger.new(log_target)
56
- @logger.level = Logger::INFO
57
- @logger.formatter = ENV['DYNO'] ? WithoutTimestamp.new : Pretty.new
58
- oldlogger.close if oldlogger && !$TESTING # don't want to close testing's STDOUT logging
59
- @logger
60
- end
61
-
62
- def self.logger
63
- defined?(@logger) ? @logger : initialize_logger
64
- end
65
-
66
- def self.logger=(log)
67
- @logger = (log ? log : Logger.new(File::NULL))
68
- end
69
-
70
- # This reopens ALL logfiles in the process that have been rotated
71
- # using logrotate(8) (without copytruncate) or similar tools.
72
- # A +File+ object is considered for reopening if it is:
73
- # 1) opened with the O_APPEND and O_WRONLY flags
74
- # 2) the current open file handle does not match its original open path
75
- # 3) unbuffered (as far as userspace buffering goes, not O_SYNC)
76
- # Returns the number of files reopened
77
- def self.reopen_logs
78
- to_reopen = []
79
- append_flags = File::WRONLY | File::APPEND
80
-
81
- ObjectSpace.each_object(File) do |fp|
82
- begin
83
- if !fp.closed? && fp.stat.file? && fp.sync && (fp.fcntl(Fcntl::F_GETFL) & append_flags) == append_flags
84
- to_reopen << fp
85
- end
86
- rescue IOError, Errno::EBADF
87
- end
88
- end
89
-
90
- nr = 0
91
- to_reopen.each do |fp|
92
- orig_st = begin
93
- fp.stat
94
- rescue IOError, Errno::EBADF
95
- next
96
- end
97
-
98
- begin
99
- b = File.stat(fp.path)
100
- next if orig_st.ino == b.ino && orig_st.dev == b.dev
101
- rescue Errno::ENOENT
102
- end
103
-
104
- begin
105
- File.open(fp.path, 'a') { |tmpfp| fp.reopen(tmpfp) }
106
- fp.sync = true
107
- nr += 1
108
- rescue IOError, Errno::EBADF
109
- # not much we can do...
110
- end
111
- end
112
- nr
113
- rescue RuntimeError => ex
114
- # RuntimeError: ObjectSpace is disabled; each_object will only work with Class, pass -X+O to enable
115
- puts "Unable to reopen logs: #{ex.message}"
116
- end
117
-
118
- def logger
119
- Sidekiq::Logging.logger
120
- end
121
- end
122
- end
@@ -1,23 +0,0 @@
1
- # frozen_string_literal: true
2
- module Sidekiq
3
- module Middleware
4
- module Server
5
- class ActiveRecord
6
-
7
- def initialize
8
- # With Rails 5+ we must use the Reloader **always**.
9
- # The reloader handles code loading and db connection management.
10
- if defined?(::Rails) && ::Rails::VERSION::MAJOR >= 5
11
- raise ArgumentError, "Rails 5 no longer needs or uses the ActiveRecord middleware."
12
- end
13
- end
14
-
15
- def call(*args)
16
- yield
17
- ensure
18
- ::ActiveRecord::Base.clear_active_connections!
19
- end
20
- end
21
- end
22
- end
23
- end
data/lib/sidekiq/util.rb DELETED
@@ -1,66 +0,0 @@
1
- # frozen_string_literal: true
2
- require 'socket'
3
- require 'securerandom'
4
- require 'sidekiq/exception_handler'
5
-
6
- module Sidekiq
7
- ##
8
- # This module is part of Sidekiq core and not intended for extensions.
9
- #
10
- module Util
11
- include ExceptionHandler
12
-
13
- EXPIRY = 60 * 60 * 24
14
-
15
- def watchdog(last_words)
16
- yield
17
- rescue Exception => ex
18
- handle_exception(ex, { context: last_words })
19
- raise ex
20
- end
21
-
22
- def safe_thread(name, &block)
23
- Thread.new do
24
- Thread.current['sidekiq_label'] = name
25
- watchdog(name, &block)
26
- end
27
- end
28
-
29
- def logger
30
- Sidekiq.logger
31
- end
32
-
33
- def redis(&block)
34
- Sidekiq.redis(&block)
35
- end
36
-
37
- def hostname
38
- ENV['DYNO'] || Socket.gethostname
39
- end
40
-
41
- def process_nonce
42
- @@process_nonce ||= SecureRandom.hex(6)
43
- end
44
-
45
- def identity
46
- @@identity ||= "#{hostname}:#{$$}:#{process_nonce}"
47
- end
48
-
49
- def fire_event(event, options={})
50
- reverse = options[:reverse]
51
- reraise = options[:reraise]
52
-
53
- arr = Sidekiq.options[:lifecycle_events][event]
54
- arr.reverse! if reverse
55
- arr.each do |block|
56
- begin
57
- block.call
58
- rescue => ex
59
- handle_exception(ex, { context: "Exception during Sidekiq lifecycle event.", event: event })
60
- raise ex if reraise
61
- end
62
- end
63
- arr.clear
64
- end
65
- end
66
- end
@@ -1,215 +0,0 @@
1
- # frozen_string_literal: true
2
- require 'sidekiq/client'
3
-
4
- module Sidekiq
5
-
6
- ##
7
- # Include this module in your worker class and you can easily create
8
- # asynchronous jobs:
9
- #
10
- # class HardWorker
11
- # include Sidekiq::Worker
12
- #
13
- # def perform(*args)
14
- # # do some work
15
- # end
16
- # end
17
- #
18
- # Then in your Rails app, you can do this:
19
- #
20
- # HardWorker.perform_async(1, 2, 3)
21
- #
22
- # Note that perform_async is a class method, perform is an instance method.
23
- module Worker
24
- attr_accessor :jid
25
-
26
- def self.included(base)
27
- raise ArgumentError, "You cannot include Sidekiq::Worker in an ActiveJob: #{base.name}" if base.ancestors.any? {|c| c.name == 'ActiveJob::Base' }
28
-
29
- base.extend(ClassMethods)
30
- base.sidekiq_class_attribute :sidekiq_options_hash
31
- base.sidekiq_class_attribute :sidekiq_retry_in_block
32
- base.sidekiq_class_attribute :sidekiq_retries_exhausted_block
33
- end
34
-
35
- def logger
36
- Sidekiq.logger
37
- end
38
-
39
- # This helper class encapsulates the set options for `set`, e.g.
40
- #
41
- # SomeWorker.set(queue: 'foo').perform_async(....)
42
- #
43
- class Setter
44
- def initialize(klass, opts)
45
- @klass = klass
46
- @opts = opts
47
- end
48
-
49
- def perform_async(*args)
50
- @klass.client_push(@opts.merge('args' => args, 'class' => @klass))
51
- end
52
-
53
- # +interval+ must be a timestamp, numeric or something that acts
54
- # numeric (like an activesupport time interval).
55
- def perform_in(interval, *args)
56
- int = interval.to_f
57
- now = Time.now.to_f
58
- ts = (int < 1_000_000_000 ? now + int : int)
59
-
60
- payload = @opts.merge('class' => @klass, 'args' => args, 'at' => ts)
61
- # Optimization to enqueue something now that is scheduled to go out now or in the past
62
- payload.delete('at') if ts <= now
63
- @klass.client_push(payload)
64
- end
65
- alias_method :perform_at, :perform_in
66
- end
67
-
68
- module ClassMethods
69
- ACCESSOR_MUTEX = Mutex.new
70
-
71
- def delay(*args)
72
- raise ArgumentError, "Do not call .delay on a Sidekiq::Worker class, call .perform_async"
73
- end
74
-
75
- def delay_for(*args)
76
- raise ArgumentError, "Do not call .delay_for on a Sidekiq::Worker class, call .perform_in"
77
- end
78
-
79
- def delay_until(*args)
80
- raise ArgumentError, "Do not call .delay_until on a Sidekiq::Worker class, call .perform_at"
81
- end
82
-
83
- def set(options)
84
- Setter.new(self, options)
85
- end
86
-
87
- def perform_async(*args)
88
- client_push('class' => self, 'args' => args)
89
- end
90
-
91
- # +interval+ must be a timestamp, numeric or something that acts
92
- # numeric (like an activesupport time interval).
93
- def perform_in(interval, *args)
94
- int = interval.to_f
95
- now = Time.now.to_f
96
- ts = (int < 1_000_000_000 ? now + int : int)
97
-
98
- item = { 'class' => self, 'args' => args, 'at' => ts }
99
-
100
- # Optimization to enqueue something now that is scheduled to go out now or in the past
101
- item.delete('at') if ts <= now
102
-
103
- client_push(item)
104
- end
105
- alias_method :perform_at, :perform_in
106
-
107
- ##
108
- # Allows customization for this type of Worker.
109
- # Legal options:
110
- #
111
- # queue - use a named queue for this Worker, default 'default'
112
- # retry - enable the RetryJobs middleware for this Worker, *true* to use the default
113
- # or *Integer* count
114
- # backtrace - whether to save any error backtrace in the retry payload to display in web UI,
115
- # can be true, false or an integer number of lines to save, default *false*
116
- # pool - use the given Redis connection pool to push this type of job to a given shard.
117
- #
118
- # In practice, any option is allowed. This is the main mechanism to configure the
119
- # options for a specific job.
120
- def sidekiq_options(opts={})
121
- # stringify
122
- self.sidekiq_options_hash = get_sidekiq_options.merge(Hash[opts.map{|k, v| [k.to_s, v]}])
123
- end
124
-
125
- def sidekiq_retry_in(&block)
126
- self.sidekiq_retry_in_block = block
127
- end
128
-
129
- def sidekiq_retries_exhausted(&block)
130
- self.sidekiq_retries_exhausted_block = block
131
- end
132
-
133
- def get_sidekiq_options # :nodoc:
134
- self.sidekiq_options_hash ||= Sidekiq.default_worker_options
135
- end
136
-
137
- def client_push(item) # :nodoc:
138
- pool = Thread.current[:sidekiq_via_pool] || get_sidekiq_options['pool'] || Sidekiq.redis_pool
139
- # stringify
140
- item.keys.each do |key|
141
- item[key.to_s] = item.delete(key)
142
- end
143
-
144
- Sidekiq::Client.new(pool).push(item)
145
- end
146
-
147
- def sidekiq_class_attribute(*attrs)
148
- instance_reader = true
149
- instance_writer = true
150
-
151
- attrs.each do |name|
152
- synchronized_getter = "__synchronized_#{name}"
153
-
154
- singleton_class.instance_eval do
155
- undef_method(name) if method_defined?(name) || private_method_defined?(name)
156
- end
157
-
158
- define_singleton_method(synchronized_getter) { nil }
159
- singleton_class.class_eval do
160
- private(synchronized_getter)
161
- end
162
-
163
- define_singleton_method(name) { ACCESSOR_MUTEX.synchronize { send synchronized_getter } }
164
-
165
- ivar = "@#{name}"
166
-
167
- singleton_class.instance_eval do
168
- m = "#{name}="
169
- undef_method(m) if method_defined?(m) || private_method_defined?(m)
170
- end
171
- define_singleton_method("#{name}=") do |val|
172
- singleton_class.class_eval do
173
- ACCESSOR_MUTEX.synchronize do
174
- undef_method(synchronized_getter) if method_defined?(synchronized_getter) || private_method_defined?(synchronized_getter)
175
- define_method(synchronized_getter) { val }
176
- end
177
- end
178
-
179
- if singleton_class?
180
- class_eval do
181
- undef_method(name) if method_defined?(name) || private_method_defined?(name)
182
- define_method(name) do
183
- if instance_variable_defined? ivar
184
- instance_variable_get ivar
185
- else
186
- singleton_class.send name
187
- end
188
- end
189
- end
190
- end
191
- val
192
- end
193
-
194
- if instance_reader
195
- undef_method(name) if method_defined?(name) || private_method_defined?(name)
196
- define_method(name) do
197
- if instance_variable_defined?(ivar)
198
- instance_variable_get ivar
199
- else
200
- self.class.public_send name
201
- end
202
- end
203
- end
204
-
205
- if instance_writer
206
- m = "#{name}="
207
- undef_method(m) if method_defined?(m) || private_method_defined?(m)
208
- attr_writer name
209
- end
210
- end
211
- end
212
-
213
- end
214
- end
215
- end