sidekiq 5.2.9 → 7.1.3

Sign up to get free protection for your applications and to get access to all the features.
Files changed (149) hide show
  1. checksums.yaml +4 -4
  2. data/Changes.md +547 -8
  3. data/LICENSE.txt +9 -0
  4. data/README.md +47 -50
  5. data/bin/sidekiq +22 -3
  6. data/bin/sidekiqload +213 -115
  7. data/bin/sidekiqmon +11 -0
  8. data/lib/generators/sidekiq/job_generator.rb +57 -0
  9. data/lib/generators/sidekiq/templates/{worker.rb.erb → job.rb.erb} +2 -2
  10. data/lib/generators/sidekiq/templates/{worker_spec.rb.erb → job_spec.rb.erb} +1 -1
  11. data/lib/generators/sidekiq/templates/{worker_test.rb.erb → job_test.rb.erb} +1 -1
  12. data/lib/sidekiq/api.rb +558 -353
  13. data/lib/sidekiq/capsule.rb +127 -0
  14. data/lib/sidekiq/cli.rb +204 -226
  15. data/lib/sidekiq/client.rb +124 -101
  16. data/lib/sidekiq/component.rb +68 -0
  17. data/lib/sidekiq/config.rb +278 -0
  18. data/lib/sidekiq/deploy.rb +62 -0
  19. data/lib/sidekiq/embedded.rb +61 -0
  20. data/lib/sidekiq/fetch.rb +49 -42
  21. data/lib/sidekiq/job.rb +374 -0
  22. data/lib/sidekiq/job_logger.rb +33 -7
  23. data/lib/sidekiq/job_retry.rb +145 -107
  24. data/lib/sidekiq/job_util.rb +107 -0
  25. data/lib/sidekiq/launcher.rb +203 -105
  26. data/lib/sidekiq/logger.rb +131 -0
  27. data/lib/sidekiq/manager.rb +43 -46
  28. data/lib/sidekiq/metrics/query.rb +153 -0
  29. data/lib/sidekiq/metrics/shared.rb +95 -0
  30. data/lib/sidekiq/metrics/tracking.rb +136 -0
  31. data/lib/sidekiq/middleware/chain.rb +113 -56
  32. data/lib/sidekiq/middleware/current_attributes.rb +95 -0
  33. data/lib/sidekiq/middleware/i18n.rb +7 -7
  34. data/lib/sidekiq/middleware/modules.rb +21 -0
  35. data/lib/sidekiq/monitor.rb +146 -0
  36. data/lib/sidekiq/paginator.rb +28 -16
  37. data/lib/sidekiq/processor.rb +122 -120
  38. data/lib/sidekiq/rails.rb +53 -38
  39. data/lib/sidekiq/redis_client_adapter.rb +96 -0
  40. data/lib/sidekiq/redis_connection.rb +38 -107
  41. data/lib/sidekiq/ring_buffer.rb +29 -0
  42. data/lib/sidekiq/scheduled.rb +111 -49
  43. data/lib/sidekiq/sd_notify.rb +149 -0
  44. data/lib/sidekiq/systemd.rb +24 -0
  45. data/lib/sidekiq/testing/inline.rb +6 -5
  46. data/lib/sidekiq/testing.rb +66 -84
  47. data/lib/sidekiq/transaction_aware_client.rb +44 -0
  48. data/lib/sidekiq/version.rb +3 -1
  49. data/lib/sidekiq/web/action.rb +15 -11
  50. data/lib/sidekiq/web/application.rb +123 -79
  51. data/lib/sidekiq/web/csrf_protection.rb +180 -0
  52. data/lib/sidekiq/web/helpers.rb +137 -106
  53. data/lib/sidekiq/web/router.rb +23 -19
  54. data/lib/sidekiq/web.rb +68 -107
  55. data/lib/sidekiq/worker_compatibility_alias.rb +13 -0
  56. data/lib/sidekiq.rb +92 -182
  57. data/sidekiq.gemspec +25 -16
  58. data/web/assets/images/apple-touch-icon.png +0 -0
  59. data/web/assets/javascripts/application.js +131 -61
  60. data/web/assets/javascripts/base-charts.js +106 -0
  61. data/web/assets/javascripts/chart.min.js +13 -0
  62. data/web/assets/javascripts/chartjs-plugin-annotation.min.js +7 -0
  63. data/web/assets/javascripts/dashboard-charts.js +168 -0
  64. data/web/assets/javascripts/dashboard.js +36 -292
  65. data/web/assets/javascripts/metrics.js +264 -0
  66. data/web/assets/stylesheets/application-dark.css +147 -0
  67. data/web/assets/stylesheets/application-rtl.css +2 -95
  68. data/web/assets/stylesheets/application.css +102 -522
  69. data/web/locales/ar.yml +71 -65
  70. data/web/locales/cs.yml +62 -62
  71. data/web/locales/da.yml +60 -53
  72. data/web/locales/de.yml +65 -53
  73. data/web/locales/el.yml +43 -24
  74. data/web/locales/en.yml +84 -66
  75. data/web/locales/es.yml +70 -54
  76. data/web/locales/fa.yml +65 -65
  77. data/web/locales/fr.yml +83 -62
  78. data/web/locales/gd.yml +99 -0
  79. data/web/locales/he.yml +65 -64
  80. data/web/locales/hi.yml +59 -59
  81. data/web/locales/it.yml +53 -53
  82. data/web/locales/ja.yml +75 -64
  83. data/web/locales/ko.yml +52 -52
  84. data/web/locales/lt.yml +83 -0
  85. data/web/locales/nb.yml +61 -61
  86. data/web/locales/nl.yml +52 -52
  87. data/web/locales/pl.yml +45 -45
  88. data/web/locales/pt-br.yml +83 -55
  89. data/web/locales/pt.yml +51 -51
  90. data/web/locales/ru.yml +68 -63
  91. data/web/locales/sv.yml +53 -53
  92. data/web/locales/ta.yml +60 -60
  93. data/web/locales/uk.yml +62 -61
  94. data/web/locales/ur.yml +64 -64
  95. data/web/locales/vi.yml +83 -0
  96. data/web/locales/zh-cn.yml +43 -16
  97. data/web/locales/zh-tw.yml +42 -8
  98. data/web/views/_footer.erb +6 -3
  99. data/web/views/_job_info.erb +21 -4
  100. data/web/views/_metrics_period_select.erb +12 -0
  101. data/web/views/_nav.erb +1 -1
  102. data/web/views/_paging.erb +2 -0
  103. data/web/views/_poll_link.erb +3 -6
  104. data/web/views/_summary.erb +7 -7
  105. data/web/views/busy.erb +75 -25
  106. data/web/views/dashboard.erb +58 -18
  107. data/web/views/dead.erb +3 -3
  108. data/web/views/layout.erb +3 -1
  109. data/web/views/metrics.erb +82 -0
  110. data/web/views/metrics_for_job.erb +68 -0
  111. data/web/views/morgue.erb +14 -15
  112. data/web/views/queue.erb +33 -24
  113. data/web/views/queues.erb +13 -3
  114. data/web/views/retries.erb +16 -17
  115. data/web/views/retry.erb +3 -3
  116. data/web/views/scheduled.erb +17 -15
  117. metadata +71 -72
  118. data/.circleci/config.yml +0 -61
  119. data/.github/contributing.md +0 -32
  120. data/.github/issue_template.md +0 -11
  121. data/.gitignore +0 -15
  122. data/.travis.yml +0 -11
  123. data/3.0-Upgrade.md +0 -70
  124. data/4.0-Upgrade.md +0 -53
  125. data/5.0-Upgrade.md +0 -56
  126. data/COMM-LICENSE +0 -97
  127. data/Ent-Changes.md +0 -238
  128. data/Gemfile +0 -23
  129. data/LICENSE +0 -9
  130. data/Pro-2.0-Upgrade.md +0 -138
  131. data/Pro-3.0-Upgrade.md +0 -44
  132. data/Pro-4.0-Upgrade.md +0 -35
  133. data/Pro-Changes.md +0 -759
  134. data/Rakefile +0 -9
  135. data/bin/sidekiqctl +0 -20
  136. data/code_of_conduct.md +0 -50
  137. data/lib/generators/sidekiq/worker_generator.rb +0 -49
  138. data/lib/sidekiq/core_ext.rb +0 -1
  139. data/lib/sidekiq/ctl.rb +0 -221
  140. data/lib/sidekiq/delay.rb +0 -42
  141. data/lib/sidekiq/exception_handler.rb +0 -29
  142. data/lib/sidekiq/extensions/action_mailer.rb +0 -57
  143. data/lib/sidekiq/extensions/active_record.rb +0 -40
  144. data/lib/sidekiq/extensions/class_methods.rb +0 -40
  145. data/lib/sidekiq/extensions/generic_proxy.rb +0 -31
  146. data/lib/sidekiq/logging.rb +0 -122
  147. data/lib/sidekiq/middleware/server/active_record.rb +0 -23
  148. data/lib/sidekiq/util.rb +0 -66
  149. data/lib/sidekiq/worker.rb +0 -220
@@ -1,122 +0,0 @@
1
- # frozen_string_literal: true
2
- require 'time'
3
- require 'logger'
4
- require 'fcntl'
5
-
6
- module Sidekiq
7
- module Logging
8
-
9
- class Pretty < Logger::Formatter
10
- SPACE = " "
11
-
12
- # Provide a call() method that returns the formatted message.
13
- def call(severity, time, program_name, message)
14
- "#{time.utc.iso8601(3)} #{::Process.pid} TID-#{Sidekiq::Logging.tid}#{context} #{severity}: #{message}\n"
15
- end
16
-
17
- def context
18
- c = Thread.current[:sidekiq_context]
19
- " #{c.join(SPACE)}" if c && c.any?
20
- end
21
- end
22
-
23
- class WithoutTimestamp < Pretty
24
- def call(severity, time, program_name, message)
25
- "#{::Process.pid} TID-#{Sidekiq::Logging.tid}#{context} #{severity}: #{message}\n"
26
- end
27
- end
28
-
29
- def self.tid
30
- Thread.current['sidekiq_tid'] ||= (Thread.current.object_id ^ ::Process.pid).to_s(36)
31
- end
32
-
33
- def self.job_hash_context(job_hash)
34
- # If we're using a wrapper class, like ActiveJob, use the "wrapped"
35
- # attribute to expose the underlying thing.
36
- klass = job_hash['wrapped'] || job_hash["class"]
37
- bid = job_hash['bid']
38
- "#{klass} JID-#{job_hash['jid']}#{" BID-#{bid}" if bid}"
39
- end
40
-
41
- def self.with_job_hash_context(job_hash, &block)
42
- with_context(job_hash_context(job_hash), &block)
43
- end
44
-
45
- def self.with_context(msg)
46
- Thread.current[:sidekiq_context] ||= []
47
- Thread.current[:sidekiq_context] << msg
48
- yield
49
- ensure
50
- Thread.current[:sidekiq_context].pop
51
- end
52
-
53
- def self.initialize_logger(log_target = STDOUT)
54
- oldlogger = defined?(@logger) ? @logger : nil
55
- @logger = Logger.new(log_target)
56
- @logger.level = Logger::INFO
57
- @logger.formatter = ENV['DYNO'] ? WithoutTimestamp.new : Pretty.new
58
- oldlogger.close if oldlogger && !$TESTING # don't want to close testing's STDOUT logging
59
- @logger
60
- end
61
-
62
- def self.logger
63
- defined?(@logger) ? @logger : initialize_logger
64
- end
65
-
66
- def self.logger=(log)
67
- @logger = (log ? log : Logger.new(File::NULL))
68
- end
69
-
70
- # This reopens ALL logfiles in the process that have been rotated
71
- # using logrotate(8) (without copytruncate) or similar tools.
72
- # A +File+ object is considered for reopening if it is:
73
- # 1) opened with the O_APPEND and O_WRONLY flags
74
- # 2) the current open file handle does not match its original open path
75
- # 3) unbuffered (as far as userspace buffering goes, not O_SYNC)
76
- # Returns the number of files reopened
77
- def self.reopen_logs
78
- to_reopen = []
79
- append_flags = File::WRONLY | File::APPEND
80
-
81
- ObjectSpace.each_object(File) do |fp|
82
- begin
83
- if !fp.closed? && fp.stat.file? && fp.sync && (fp.fcntl(Fcntl::F_GETFL) & append_flags) == append_flags
84
- to_reopen << fp
85
- end
86
- rescue IOError, Errno::EBADF
87
- end
88
- end
89
-
90
- nr = 0
91
- to_reopen.each do |fp|
92
- orig_st = begin
93
- fp.stat
94
- rescue IOError, Errno::EBADF
95
- next
96
- end
97
-
98
- begin
99
- b = File.stat(fp.path)
100
- next if orig_st.ino == b.ino && orig_st.dev == b.dev
101
- rescue Errno::ENOENT
102
- end
103
-
104
- begin
105
- File.open(fp.path, 'a') { |tmpfp| fp.reopen(tmpfp) }
106
- fp.sync = true
107
- nr += 1
108
- rescue IOError, Errno::EBADF
109
- # not much we can do...
110
- end
111
- end
112
- nr
113
- rescue RuntimeError => ex
114
- # RuntimeError: ObjectSpace is disabled; each_object will only work with Class, pass -X+O to enable
115
- puts "Unable to reopen logs: #{ex.message}"
116
- end
117
-
118
- def logger
119
- Sidekiq::Logging.logger
120
- end
121
- end
122
- end
@@ -1,23 +0,0 @@
1
- # frozen_string_literal: true
2
- module Sidekiq
3
- module Middleware
4
- module Server
5
- class ActiveRecord
6
-
7
- def initialize
8
- # With Rails 5+ we must use the Reloader **always**.
9
- # The reloader handles code loading and db connection management.
10
- if defined?(::Rails) && defined?(::Rails::VERSION) && ::Rails::VERSION::MAJOR >= 5
11
- raise ArgumentError, "Rails 5 no longer needs or uses the ActiveRecord middleware."
12
- end
13
- end
14
-
15
- def call(*args)
16
- yield
17
- ensure
18
- ::ActiveRecord::Base.clear_active_connections!
19
- end
20
- end
21
- end
22
- end
23
- end
data/lib/sidekiq/util.rb DELETED
@@ -1,66 +0,0 @@
1
- # frozen_string_literal: true
2
- require 'socket'
3
- require 'securerandom'
4
- require 'sidekiq/exception_handler'
5
-
6
- module Sidekiq
7
- ##
8
- # This module is part of Sidekiq core and not intended for extensions.
9
- #
10
- module Util
11
- include ExceptionHandler
12
-
13
- EXPIRY = 60 * 60 * 24
14
-
15
- def watchdog(last_words)
16
- yield
17
- rescue Exception => ex
18
- handle_exception(ex, { context: last_words })
19
- raise ex
20
- end
21
-
22
- def safe_thread(name, &block)
23
- Thread.new do
24
- Thread.current['sidekiq_label'] = name
25
- watchdog(name, &block)
26
- end
27
- end
28
-
29
- def logger
30
- Sidekiq.logger
31
- end
32
-
33
- def redis(&block)
34
- Sidekiq.redis(&block)
35
- end
36
-
37
- def hostname
38
- ENV['DYNO'] || Socket.gethostname
39
- end
40
-
41
- def process_nonce
42
- @@process_nonce ||= SecureRandom.hex(6)
43
- end
44
-
45
- def identity
46
- @@identity ||= "#{hostname}:#{$$}:#{process_nonce}"
47
- end
48
-
49
- def fire_event(event, options={})
50
- reverse = options[:reverse]
51
- reraise = options[:reraise]
52
-
53
- arr = Sidekiq.options[:lifecycle_events][event]
54
- arr.reverse! if reverse
55
- arr.each do |block|
56
- begin
57
- block.call
58
- rescue => ex
59
- handle_exception(ex, { context: "Exception during Sidekiq lifecycle event.", event: event })
60
- raise ex if reraise
61
- end
62
- end
63
- arr.clear
64
- end
65
- end
66
- end
@@ -1,220 +0,0 @@
1
- # frozen_string_literal: true
2
- require 'sidekiq/client'
3
-
4
- module Sidekiq
5
-
6
- ##
7
- # Include this module in your worker class and you can easily create
8
- # asynchronous jobs:
9
- #
10
- # class HardWorker
11
- # include Sidekiq::Worker
12
- #
13
- # def perform(*args)
14
- # # do some work
15
- # end
16
- # end
17
- #
18
- # Then in your Rails app, you can do this:
19
- #
20
- # HardWorker.perform_async(1, 2, 3)
21
- #
22
- # Note that perform_async is a class method, perform is an instance method.
23
- module Worker
24
- attr_accessor :jid
25
-
26
- def self.included(base)
27
- raise ArgumentError, "You cannot include Sidekiq::Worker in an ActiveJob: #{base.name}" if base.ancestors.any? {|c| c.name == 'ActiveJob::Base' }
28
-
29
- base.extend(ClassMethods)
30
- base.sidekiq_class_attribute :sidekiq_options_hash
31
- base.sidekiq_class_attribute :sidekiq_retry_in_block
32
- base.sidekiq_class_attribute :sidekiq_retries_exhausted_block
33
- end
34
-
35
- def logger
36
- Sidekiq.logger
37
- end
38
-
39
- # This helper class encapsulates the set options for `set`, e.g.
40
- #
41
- # SomeWorker.set(queue: 'foo').perform_async(....)
42
- #
43
- class Setter
44
- def initialize(klass, opts)
45
- @klass = klass
46
- @opts = opts
47
- end
48
-
49
- def set(options)
50
- @opts.merge!(options)
51
- self
52
- end
53
-
54
- def perform_async(*args)
55
- @klass.client_push(@opts.merge('args' => args, 'class' => @klass))
56
- end
57
-
58
- # +interval+ must be a timestamp, numeric or something that acts
59
- # numeric (like an activesupport time interval).
60
- def perform_in(interval, *args)
61
- int = interval.to_f
62
- now = Time.now.to_f
63
- ts = (int < 1_000_000_000 ? now + int : int)
64
-
65
- payload = @opts.merge('class' => @klass, 'args' => args, 'at' => ts)
66
- # Optimization to enqueue something now that is scheduled to go out now or in the past
67
- payload.delete('at') if ts <= now
68
- @klass.client_push(payload)
69
- end
70
- alias_method :perform_at, :perform_in
71
- end
72
-
73
- module ClassMethods
74
- ACCESSOR_MUTEX = Mutex.new
75
-
76
- def delay(*args)
77
- raise ArgumentError, "Do not call .delay on a Sidekiq::Worker class, call .perform_async"
78
- end
79
-
80
- def delay_for(*args)
81
- raise ArgumentError, "Do not call .delay_for on a Sidekiq::Worker class, call .perform_in"
82
- end
83
-
84
- def delay_until(*args)
85
- raise ArgumentError, "Do not call .delay_until on a Sidekiq::Worker class, call .perform_at"
86
- end
87
-
88
- def set(options)
89
- Setter.new(self, options)
90
- end
91
-
92
- def perform_async(*args)
93
- client_push('class' => self, 'args' => args)
94
- end
95
-
96
- # +interval+ must be a timestamp, numeric or something that acts
97
- # numeric (like an activesupport time interval).
98
- def perform_in(interval, *args)
99
- int = interval.to_f
100
- now = Time.now.to_f
101
- ts = (int < 1_000_000_000 ? now + int : int)
102
-
103
- item = { 'class' => self, 'args' => args, 'at' => ts }
104
-
105
- # Optimization to enqueue something now that is scheduled to go out now or in the past
106
- item.delete('at') if ts <= now
107
-
108
- client_push(item)
109
- end
110
- alias_method :perform_at, :perform_in
111
-
112
- ##
113
- # Allows customization for this type of Worker.
114
- # Legal options:
115
- #
116
- # queue - use a named queue for this Worker, default 'default'
117
- # retry - enable the RetryJobs middleware for this Worker, *true* to use the default
118
- # or *Integer* count
119
- # backtrace - whether to save any error backtrace in the retry payload to display in web UI,
120
- # can be true, false or an integer number of lines to save, default *false*
121
- # pool - use the given Redis connection pool to push this type of job to a given shard.
122
- #
123
- # In practice, any option is allowed. This is the main mechanism to configure the
124
- # options for a specific job.
125
- def sidekiq_options(opts={})
126
- # stringify
127
- self.sidekiq_options_hash = get_sidekiq_options.merge(Hash[opts.map{|k, v| [k.to_s, v]}])
128
- end
129
-
130
- def sidekiq_retry_in(&block)
131
- self.sidekiq_retry_in_block = block
132
- end
133
-
134
- def sidekiq_retries_exhausted(&block)
135
- self.sidekiq_retries_exhausted_block = block
136
- end
137
-
138
- def get_sidekiq_options # :nodoc:
139
- self.sidekiq_options_hash ||= Sidekiq.default_worker_options
140
- end
141
-
142
- def client_push(item) # :nodoc:
143
- pool = Thread.current[:sidekiq_via_pool] || get_sidekiq_options['pool'] || Sidekiq.redis_pool
144
- # stringify
145
- item.keys.each do |key|
146
- item[key.to_s] = item.delete(key)
147
- end
148
-
149
- Sidekiq::Client.new(pool).push(item)
150
- end
151
-
152
- def sidekiq_class_attribute(*attrs)
153
- instance_reader = true
154
- instance_writer = true
155
-
156
- attrs.each do |name|
157
- synchronized_getter = "__synchronized_#{name}"
158
-
159
- singleton_class.instance_eval do
160
- undef_method(name) if method_defined?(name) || private_method_defined?(name)
161
- end
162
-
163
- define_singleton_method(synchronized_getter) { nil }
164
- singleton_class.class_eval do
165
- private(synchronized_getter)
166
- end
167
-
168
- define_singleton_method(name) { ACCESSOR_MUTEX.synchronize { send synchronized_getter } }
169
-
170
- ivar = "@#{name}"
171
-
172
- singleton_class.instance_eval do
173
- m = "#{name}="
174
- undef_method(m) if method_defined?(m) || private_method_defined?(m)
175
- end
176
- define_singleton_method("#{name}=") do |val|
177
- singleton_class.class_eval do
178
- ACCESSOR_MUTEX.synchronize do
179
- undef_method(synchronized_getter) if method_defined?(synchronized_getter) || private_method_defined?(synchronized_getter)
180
- define_method(synchronized_getter) { val }
181
- end
182
- end
183
-
184
- if singleton_class?
185
- class_eval do
186
- undef_method(name) if method_defined?(name) || private_method_defined?(name)
187
- define_method(name) do
188
- if instance_variable_defined? ivar
189
- instance_variable_get ivar
190
- else
191
- singleton_class.send name
192
- end
193
- end
194
- end
195
- end
196
- val
197
- end
198
-
199
- if instance_reader
200
- undef_method(name) if method_defined?(name) || private_method_defined?(name)
201
- define_method(name) do
202
- if instance_variable_defined?(ivar)
203
- instance_variable_get ivar
204
- else
205
- self.class.public_send name
206
- end
207
- end
208
- end
209
-
210
- if instance_writer
211
- m = "#{name}="
212
- undef_method(m) if method_defined?(m) || private_method_defined?(m)
213
- attr_writer name
214
- end
215
- end
216
- end
217
-
218
- end
219
- end
220
- end