sidekiq 5.2.4 → 7.2.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (153) hide show
  1. checksums.yaml +4 -4
  2. data/Changes.md +672 -8
  3. data/LICENSE.txt +9 -0
  4. data/README.md +48 -51
  5. data/bin/multi_queue_bench +271 -0
  6. data/bin/sidekiq +22 -3
  7. data/bin/sidekiqload +213 -115
  8. data/bin/sidekiqmon +11 -0
  9. data/lib/generators/sidekiq/job_generator.rb +57 -0
  10. data/lib/generators/sidekiq/templates/{worker.rb.erb → job.rb.erb} +2 -2
  11. data/lib/generators/sidekiq/templates/{worker_spec.rb.erb → job_spec.rb.erb} +1 -1
  12. data/lib/generators/sidekiq/templates/{worker_test.rb.erb → job_test.rb.erb} +1 -1
  13. data/lib/sidekiq/api.rb +623 -352
  14. data/lib/sidekiq/capsule.rb +127 -0
  15. data/lib/sidekiq/cli.rb +214 -229
  16. data/lib/sidekiq/client.rb +127 -102
  17. data/lib/sidekiq/component.rb +68 -0
  18. data/lib/sidekiq/config.rb +287 -0
  19. data/lib/sidekiq/deploy.rb +62 -0
  20. data/lib/sidekiq/embedded.rb +61 -0
  21. data/lib/sidekiq/fetch.rb +49 -42
  22. data/lib/sidekiq/job.rb +374 -0
  23. data/lib/sidekiq/job_logger.rb +33 -7
  24. data/lib/sidekiq/job_retry.rb +157 -108
  25. data/lib/sidekiq/job_util.rb +107 -0
  26. data/lib/sidekiq/launcher.rb +206 -106
  27. data/lib/sidekiq/logger.rb +131 -0
  28. data/lib/sidekiq/manager.rb +43 -46
  29. data/lib/sidekiq/metrics/query.rb +156 -0
  30. data/lib/sidekiq/metrics/shared.rb +95 -0
  31. data/lib/sidekiq/metrics/tracking.rb +140 -0
  32. data/lib/sidekiq/middleware/chain.rb +113 -56
  33. data/lib/sidekiq/middleware/current_attributes.rb +95 -0
  34. data/lib/sidekiq/middleware/i18n.rb +7 -7
  35. data/lib/sidekiq/middleware/modules.rb +21 -0
  36. data/lib/sidekiq/monitor.rb +146 -0
  37. data/lib/sidekiq/paginator.rb +28 -16
  38. data/lib/sidekiq/processor.rb +126 -117
  39. data/lib/sidekiq/rails.rb +52 -38
  40. data/lib/sidekiq/redis_client_adapter.rb +111 -0
  41. data/lib/sidekiq/redis_connection.rb +41 -112
  42. data/lib/sidekiq/ring_buffer.rb +29 -0
  43. data/lib/sidekiq/scheduled.rb +112 -50
  44. data/lib/sidekiq/sd_notify.rb +149 -0
  45. data/lib/sidekiq/systemd.rb +24 -0
  46. data/lib/sidekiq/testing/inline.rb +6 -5
  47. data/lib/sidekiq/testing.rb +91 -90
  48. data/lib/sidekiq/transaction_aware_client.rb +51 -0
  49. data/lib/sidekiq/version.rb +3 -1
  50. data/lib/sidekiq/web/action.rb +20 -11
  51. data/lib/sidekiq/web/application.rb +202 -80
  52. data/lib/sidekiq/web/csrf_protection.rb +183 -0
  53. data/lib/sidekiq/web/helpers.rb +165 -114
  54. data/lib/sidekiq/web/router.rb +23 -19
  55. data/lib/sidekiq/web.rb +68 -107
  56. data/lib/sidekiq/worker_compatibility_alias.rb +13 -0
  57. data/lib/sidekiq.rb +92 -182
  58. data/sidekiq.gemspec +25 -16
  59. data/web/assets/images/apple-touch-icon.png +0 -0
  60. data/web/assets/javascripts/application.js +152 -61
  61. data/web/assets/javascripts/base-charts.js +106 -0
  62. data/web/assets/javascripts/chart.min.js +13 -0
  63. data/web/assets/javascripts/chartjs-plugin-annotation.min.js +7 -0
  64. data/web/assets/javascripts/dashboard-charts.js +182 -0
  65. data/web/assets/javascripts/dashboard.js +35 -293
  66. data/web/assets/javascripts/metrics.js +298 -0
  67. data/web/assets/stylesheets/application-dark.css +147 -0
  68. data/web/assets/stylesheets/application-rtl.css +10 -93
  69. data/web/assets/stylesheets/application.css +124 -522
  70. data/web/assets/stylesheets/bootstrap.css +1 -1
  71. data/web/locales/ar.yml +71 -65
  72. data/web/locales/cs.yml +62 -62
  73. data/web/locales/da.yml +60 -53
  74. data/web/locales/de.yml +65 -53
  75. data/web/locales/el.yml +43 -24
  76. data/web/locales/en.yml +86 -66
  77. data/web/locales/es.yml +70 -54
  78. data/web/locales/fa.yml +65 -65
  79. data/web/locales/fr.yml +83 -62
  80. data/web/locales/gd.yml +99 -0
  81. data/web/locales/he.yml +65 -64
  82. data/web/locales/hi.yml +59 -59
  83. data/web/locales/it.yml +53 -53
  84. data/web/locales/ja.yml +75 -64
  85. data/web/locales/ko.yml +52 -52
  86. data/web/locales/lt.yml +83 -0
  87. data/web/locales/nb.yml +61 -61
  88. data/web/locales/nl.yml +52 -52
  89. data/web/locales/pl.yml +45 -45
  90. data/web/locales/pt-br.yml +83 -55
  91. data/web/locales/pt.yml +51 -51
  92. data/web/locales/ru.yml +68 -63
  93. data/web/locales/sv.yml +53 -53
  94. data/web/locales/ta.yml +60 -60
  95. data/web/locales/uk.yml +62 -61
  96. data/web/locales/ur.yml +64 -64
  97. data/web/locales/vi.yml +83 -0
  98. data/web/locales/zh-cn.yml +43 -16
  99. data/web/locales/zh-tw.yml +42 -8
  100. data/web/views/_footer.erb +18 -3
  101. data/web/views/_job_info.erb +21 -4
  102. data/web/views/_metrics_period_select.erb +12 -0
  103. data/web/views/_nav.erb +1 -1
  104. data/web/views/_paging.erb +2 -0
  105. data/web/views/_poll_link.erb +3 -6
  106. data/web/views/_summary.erb +7 -7
  107. data/web/views/busy.erb +79 -29
  108. data/web/views/dashboard.erb +48 -18
  109. data/web/views/dead.erb +3 -3
  110. data/web/views/filtering.erb +7 -0
  111. data/web/views/layout.erb +3 -1
  112. data/web/views/metrics.erb +91 -0
  113. data/web/views/metrics_for_job.erb +59 -0
  114. data/web/views/morgue.erb +14 -15
  115. data/web/views/queue.erb +33 -24
  116. data/web/views/queues.erb +19 -5
  117. data/web/views/retries.erb +16 -17
  118. data/web/views/retry.erb +3 -3
  119. data/web/views/scheduled.erb +17 -15
  120. metadata +71 -72
  121. data/.github/contributing.md +0 -32
  122. data/.github/issue_template.md +0 -11
  123. data/.gitignore +0 -15
  124. data/.travis.yml +0 -17
  125. data/3.0-Upgrade.md +0 -70
  126. data/4.0-Upgrade.md +0 -53
  127. data/5.0-Upgrade.md +0 -56
  128. data/Appraisals +0 -9
  129. data/COMM-LICENSE +0 -95
  130. data/Ent-Changes.md +0 -225
  131. data/Gemfile +0 -29
  132. data/LICENSE +0 -9
  133. data/Pro-2.0-Upgrade.md +0 -138
  134. data/Pro-3.0-Upgrade.md +0 -44
  135. data/Pro-4.0-Upgrade.md +0 -35
  136. data/Pro-Changes.md +0 -752
  137. data/Rakefile +0 -9
  138. data/bin/sidekiqctl +0 -237
  139. data/code_of_conduct.md +0 -50
  140. data/gemfiles/rails_4.gemfile +0 -31
  141. data/gemfiles/rails_5.gemfile +0 -31
  142. data/lib/generators/sidekiq/worker_generator.rb +0 -49
  143. data/lib/sidekiq/core_ext.rb +0 -1
  144. data/lib/sidekiq/delay.rb +0 -42
  145. data/lib/sidekiq/exception_handler.rb +0 -29
  146. data/lib/sidekiq/extensions/action_mailer.rb +0 -57
  147. data/lib/sidekiq/extensions/active_record.rb +0 -40
  148. data/lib/sidekiq/extensions/class_methods.rb +0 -40
  149. data/lib/sidekiq/extensions/generic_proxy.rb +0 -31
  150. data/lib/sidekiq/logging.rb +0 -122
  151. data/lib/sidekiq/middleware/server/active_record.rb +0 -23
  152. data/lib/sidekiq/util.rb +0 -66
  153. data/lib/sidekiq/worker.rb +0 -215
@@ -1,122 +0,0 @@
1
- # frozen_string_literal: true
2
- require 'time'
3
- require 'logger'
4
- require 'fcntl'
5
-
6
- module Sidekiq
7
- module Logging
8
-
9
- class Pretty < Logger::Formatter
10
- SPACE = " "
11
-
12
- # Provide a call() method that returns the formatted message.
13
- def call(severity, time, program_name, message)
14
- "#{time.utc.iso8601(3)} #{::Process.pid} TID-#{Sidekiq::Logging.tid}#{context} #{severity}: #{message}\n"
15
- end
16
-
17
- def context
18
- c = Thread.current[:sidekiq_context]
19
- " #{c.join(SPACE)}" if c && c.any?
20
- end
21
- end
22
-
23
- class WithoutTimestamp < Pretty
24
- def call(severity, time, program_name, message)
25
- "#{::Process.pid} TID-#{Sidekiq::Logging.tid}#{context} #{severity}: #{message}\n"
26
- end
27
- end
28
-
29
- def self.tid
30
- Thread.current['sidekiq_tid'] ||= (Thread.current.object_id ^ ::Process.pid).to_s(36)
31
- end
32
-
33
- def self.job_hash_context(job_hash)
34
- # If we're using a wrapper class, like ActiveJob, use the "wrapped"
35
- # attribute to expose the underlying thing.
36
- klass = job_hash['wrapped'] || job_hash["class"]
37
- bid = job_hash['bid']
38
- "#{klass} JID-#{job_hash['jid']}#{" BID-#{bid}" if bid}"
39
- end
40
-
41
- def self.with_job_hash_context(job_hash, &block)
42
- with_context(job_hash_context(job_hash), &block)
43
- end
44
-
45
- def self.with_context(msg)
46
- Thread.current[:sidekiq_context] ||= []
47
- Thread.current[:sidekiq_context] << msg
48
- yield
49
- ensure
50
- Thread.current[:sidekiq_context].pop
51
- end
52
-
53
- def self.initialize_logger(log_target = STDOUT)
54
- oldlogger = defined?(@logger) ? @logger : nil
55
- @logger = Logger.new(log_target)
56
- @logger.level = Logger::INFO
57
- @logger.formatter = ENV['DYNO'] ? WithoutTimestamp.new : Pretty.new
58
- oldlogger.close if oldlogger && !$TESTING # don't want to close testing's STDOUT logging
59
- @logger
60
- end
61
-
62
- def self.logger
63
- defined?(@logger) ? @logger : initialize_logger
64
- end
65
-
66
- def self.logger=(log)
67
- @logger = (log ? log : Logger.new(File::NULL))
68
- end
69
-
70
- # This reopens ALL logfiles in the process that have been rotated
71
- # using logrotate(8) (without copytruncate) or similar tools.
72
- # A +File+ object is considered for reopening if it is:
73
- # 1) opened with the O_APPEND and O_WRONLY flags
74
- # 2) the current open file handle does not match its original open path
75
- # 3) unbuffered (as far as userspace buffering goes, not O_SYNC)
76
- # Returns the number of files reopened
77
- def self.reopen_logs
78
- to_reopen = []
79
- append_flags = File::WRONLY | File::APPEND
80
-
81
- ObjectSpace.each_object(File) do |fp|
82
- begin
83
- if !fp.closed? && fp.stat.file? && fp.sync && (fp.fcntl(Fcntl::F_GETFL) & append_flags) == append_flags
84
- to_reopen << fp
85
- end
86
- rescue IOError, Errno::EBADF
87
- end
88
- end
89
-
90
- nr = 0
91
- to_reopen.each do |fp|
92
- orig_st = begin
93
- fp.stat
94
- rescue IOError, Errno::EBADF
95
- next
96
- end
97
-
98
- begin
99
- b = File.stat(fp.path)
100
- next if orig_st.ino == b.ino && orig_st.dev == b.dev
101
- rescue Errno::ENOENT
102
- end
103
-
104
- begin
105
- File.open(fp.path, 'a') { |tmpfp| fp.reopen(tmpfp) }
106
- fp.sync = true
107
- nr += 1
108
- rescue IOError, Errno::EBADF
109
- # not much we can do...
110
- end
111
- end
112
- nr
113
- rescue RuntimeError => ex
114
- # RuntimeError: ObjectSpace is disabled; each_object will only work with Class, pass -X+O to enable
115
- puts "Unable to reopen logs: #{ex.message}"
116
- end
117
-
118
- def logger
119
- Sidekiq::Logging.logger
120
- end
121
- end
122
- end
@@ -1,23 +0,0 @@
1
- # frozen_string_literal: true
2
- module Sidekiq
3
- module Middleware
4
- module Server
5
- class ActiveRecord
6
-
7
- def initialize
8
- # With Rails 5+ we must use the Reloader **always**.
9
- # The reloader handles code loading and db connection management.
10
- if defined?(::Rails) && ::Rails::VERSION::MAJOR >= 5
11
- raise ArgumentError, "Rails 5 no longer needs or uses the ActiveRecord middleware."
12
- end
13
- end
14
-
15
- def call(*args)
16
- yield
17
- ensure
18
- ::ActiveRecord::Base.clear_active_connections!
19
- end
20
- end
21
- end
22
- end
23
- end
data/lib/sidekiq/util.rb DELETED
@@ -1,66 +0,0 @@
1
- # frozen_string_literal: true
2
- require 'socket'
3
- require 'securerandom'
4
- require 'sidekiq/exception_handler'
5
-
6
- module Sidekiq
7
- ##
8
- # This module is part of Sidekiq core and not intended for extensions.
9
- #
10
- module Util
11
- include ExceptionHandler
12
-
13
- EXPIRY = 60 * 60 * 24
14
-
15
- def watchdog(last_words)
16
- yield
17
- rescue Exception => ex
18
- handle_exception(ex, { context: last_words })
19
- raise ex
20
- end
21
-
22
- def safe_thread(name, &block)
23
- Thread.new do
24
- Thread.current['sidekiq_label'] = name
25
- watchdog(name, &block)
26
- end
27
- end
28
-
29
- def logger
30
- Sidekiq.logger
31
- end
32
-
33
- def redis(&block)
34
- Sidekiq.redis(&block)
35
- end
36
-
37
- def hostname
38
- ENV['DYNO'] || Socket.gethostname
39
- end
40
-
41
- def process_nonce
42
- @@process_nonce ||= SecureRandom.hex(6)
43
- end
44
-
45
- def identity
46
- @@identity ||= "#{hostname}:#{$$}:#{process_nonce}"
47
- end
48
-
49
- def fire_event(event, options={})
50
- reverse = options[:reverse]
51
- reraise = options[:reraise]
52
-
53
- arr = Sidekiq.options[:lifecycle_events][event]
54
- arr.reverse! if reverse
55
- arr.each do |block|
56
- begin
57
- block.call
58
- rescue => ex
59
- handle_exception(ex, { context: "Exception during Sidekiq lifecycle event.", event: event })
60
- raise ex if reraise
61
- end
62
- end
63
- arr.clear
64
- end
65
- end
66
- end
@@ -1,215 +0,0 @@
1
- # frozen_string_literal: true
2
- require 'sidekiq/client'
3
-
4
- module Sidekiq
5
-
6
- ##
7
- # Include this module in your worker class and you can easily create
8
- # asynchronous jobs:
9
- #
10
- # class HardWorker
11
- # include Sidekiq::Worker
12
- #
13
- # def perform(*args)
14
- # # do some work
15
- # end
16
- # end
17
- #
18
- # Then in your Rails app, you can do this:
19
- #
20
- # HardWorker.perform_async(1, 2, 3)
21
- #
22
- # Note that perform_async is a class method, perform is an instance method.
23
- module Worker
24
- attr_accessor :jid
25
-
26
- def self.included(base)
27
- raise ArgumentError, "You cannot include Sidekiq::Worker in an ActiveJob: #{base.name}" if base.ancestors.any? {|c| c.name == 'ActiveJob::Base' }
28
-
29
- base.extend(ClassMethods)
30
- base.sidekiq_class_attribute :sidekiq_options_hash
31
- base.sidekiq_class_attribute :sidekiq_retry_in_block
32
- base.sidekiq_class_attribute :sidekiq_retries_exhausted_block
33
- end
34
-
35
- def logger
36
- Sidekiq.logger
37
- end
38
-
39
- # This helper class encapsulates the set options for `set`, e.g.
40
- #
41
- # SomeWorker.set(queue: 'foo').perform_async(....)
42
- #
43
- class Setter
44
- def initialize(klass, opts)
45
- @klass = klass
46
- @opts = opts
47
- end
48
-
49
- def perform_async(*args)
50
- @klass.client_push(@opts.merge('args' => args, 'class' => @klass))
51
- end
52
-
53
- # +interval+ must be a timestamp, numeric or something that acts
54
- # numeric (like an activesupport time interval).
55
- def perform_in(interval, *args)
56
- int = interval.to_f
57
- now = Time.now.to_f
58
- ts = (int < 1_000_000_000 ? now + int : int)
59
-
60
- payload = @opts.merge('class' => @klass, 'args' => args, 'at' => ts)
61
- # Optimization to enqueue something now that is scheduled to go out now or in the past
62
- payload.delete('at') if ts <= now
63
- @klass.client_push(payload)
64
- end
65
- alias_method :perform_at, :perform_in
66
- end
67
-
68
- module ClassMethods
69
- ACCESSOR_MUTEX = Mutex.new
70
-
71
- def delay(*args)
72
- raise ArgumentError, "Do not call .delay on a Sidekiq::Worker class, call .perform_async"
73
- end
74
-
75
- def delay_for(*args)
76
- raise ArgumentError, "Do not call .delay_for on a Sidekiq::Worker class, call .perform_in"
77
- end
78
-
79
- def delay_until(*args)
80
- raise ArgumentError, "Do not call .delay_until on a Sidekiq::Worker class, call .perform_at"
81
- end
82
-
83
- def set(options)
84
- Setter.new(self, options)
85
- end
86
-
87
- def perform_async(*args)
88
- client_push('class' => self, 'args' => args)
89
- end
90
-
91
- # +interval+ must be a timestamp, numeric or something that acts
92
- # numeric (like an activesupport time interval).
93
- def perform_in(interval, *args)
94
- int = interval.to_f
95
- now = Time.now.to_f
96
- ts = (int < 1_000_000_000 ? now + int : int)
97
-
98
- item = { 'class' => self, 'args' => args, 'at' => ts }
99
-
100
- # Optimization to enqueue something now that is scheduled to go out now or in the past
101
- item.delete('at') if ts <= now
102
-
103
- client_push(item)
104
- end
105
- alias_method :perform_at, :perform_in
106
-
107
- ##
108
- # Allows customization for this type of Worker.
109
- # Legal options:
110
- #
111
- # queue - use a named queue for this Worker, default 'default'
112
- # retry - enable the RetryJobs middleware for this Worker, *true* to use the default
113
- # or *Integer* count
114
- # backtrace - whether to save any error backtrace in the retry payload to display in web UI,
115
- # can be true, false or an integer number of lines to save, default *false*
116
- # pool - use the given Redis connection pool to push this type of job to a given shard.
117
- #
118
- # In practice, any option is allowed. This is the main mechanism to configure the
119
- # options for a specific job.
120
- def sidekiq_options(opts={})
121
- # stringify
122
- self.sidekiq_options_hash = get_sidekiq_options.merge(Hash[opts.map{|k, v| [k.to_s, v]}])
123
- end
124
-
125
- def sidekiq_retry_in(&block)
126
- self.sidekiq_retry_in_block = block
127
- end
128
-
129
- def sidekiq_retries_exhausted(&block)
130
- self.sidekiq_retries_exhausted_block = block
131
- end
132
-
133
- def get_sidekiq_options # :nodoc:
134
- self.sidekiq_options_hash ||= Sidekiq.default_worker_options
135
- end
136
-
137
- def client_push(item) # :nodoc:
138
- pool = Thread.current[:sidekiq_via_pool] || get_sidekiq_options['pool'] || Sidekiq.redis_pool
139
- # stringify
140
- item.keys.each do |key|
141
- item[key.to_s] = item.delete(key)
142
- end
143
-
144
- Sidekiq::Client.new(pool).push(item)
145
- end
146
-
147
- def sidekiq_class_attribute(*attrs)
148
- instance_reader = true
149
- instance_writer = true
150
-
151
- attrs.each do |name|
152
- synchronized_getter = "__synchronized_#{name}"
153
-
154
- singleton_class.instance_eval do
155
- undef_method(name) if method_defined?(name) || private_method_defined?(name)
156
- end
157
-
158
- define_singleton_method(synchronized_getter) { nil }
159
- singleton_class.class_eval do
160
- private(synchronized_getter)
161
- end
162
-
163
- define_singleton_method(name) { ACCESSOR_MUTEX.synchronize { send synchronized_getter } }
164
-
165
- ivar = "@#{name}"
166
-
167
- singleton_class.instance_eval do
168
- m = "#{name}="
169
- undef_method(m) if method_defined?(m) || private_method_defined?(m)
170
- end
171
- define_singleton_method("#{name}=") do |val|
172
- singleton_class.class_eval do
173
- ACCESSOR_MUTEX.synchronize do
174
- undef_method(synchronized_getter) if method_defined?(synchronized_getter) || private_method_defined?(synchronized_getter)
175
- define_method(synchronized_getter) { val }
176
- end
177
- end
178
-
179
- if singleton_class?
180
- class_eval do
181
- undef_method(name) if method_defined?(name) || private_method_defined?(name)
182
- define_method(name) do
183
- if instance_variable_defined? ivar
184
- instance_variable_get ivar
185
- else
186
- singleton_class.send name
187
- end
188
- end
189
- end
190
- end
191
- val
192
- end
193
-
194
- if instance_reader
195
- undef_method(name) if method_defined?(name) || private_method_defined?(name)
196
- define_method(name) do
197
- if instance_variable_defined?(ivar)
198
- instance_variable_get ivar
199
- else
200
- self.class.public_send name
201
- end
202
- end
203
- end
204
-
205
- if instance_writer
206
- m = "#{name}="
207
- undef_method(m) if method_defined?(m) || private_method_defined?(m)
208
- attr_writer name
209
- end
210
- end
211
- end
212
-
213
- end
214
- end
215
- end