sidekiq 6.3.1 → 7.1.5

Sign up to get free protection for your applications and to get access to all the features.
Files changed (120) hide show
  1. checksums.yaml +4 -4
  2. data/Changes.md +285 -11
  3. data/LICENSE.txt +9 -0
  4. data/README.md +47 -34
  5. data/bin/sidekiq +4 -9
  6. data/bin/sidekiqload +207 -117
  7. data/bin/sidekiqmon +4 -1
  8. data/lib/generators/sidekiq/job_generator.rb +57 -0
  9. data/lib/generators/sidekiq/templates/{worker.rb.erb → job.rb.erb} +2 -2
  10. data/lib/generators/sidekiq/templates/{worker_spec.rb.erb → job_spec.rb.erb} +1 -1
  11. data/lib/generators/sidekiq/templates/{worker_test.rb.erb → job_test.rb.erb} +1 -1
  12. data/lib/sidekiq/api.rb +333 -190
  13. data/lib/sidekiq/capsule.rb +127 -0
  14. data/lib/sidekiq/cli.rb +86 -80
  15. data/lib/sidekiq/client.rb +104 -95
  16. data/lib/sidekiq/{util.rb → component.rb} +14 -41
  17. data/lib/sidekiq/config.rb +282 -0
  18. data/lib/sidekiq/deploy.rb +62 -0
  19. data/lib/sidekiq/embedded.rb +61 -0
  20. data/lib/sidekiq/fetch.rb +23 -24
  21. data/lib/sidekiq/job.rb +371 -10
  22. data/lib/sidekiq/job_logger.rb +16 -28
  23. data/lib/sidekiq/job_retry.rb +99 -58
  24. data/lib/sidekiq/job_util.rb +107 -0
  25. data/lib/sidekiq/launcher.rb +103 -95
  26. data/lib/sidekiq/logger.rb +9 -44
  27. data/lib/sidekiq/manager.rb +40 -41
  28. data/lib/sidekiq/metrics/query.rb +153 -0
  29. data/lib/sidekiq/metrics/shared.rb +95 -0
  30. data/lib/sidekiq/metrics/tracking.rb +136 -0
  31. data/lib/sidekiq/middleware/chain.rb +96 -51
  32. data/lib/sidekiq/middleware/current_attributes.rb +59 -16
  33. data/lib/sidekiq/middleware/i18n.rb +6 -4
  34. data/lib/sidekiq/middleware/modules.rb +21 -0
  35. data/lib/sidekiq/monitor.rb +17 -4
  36. data/lib/sidekiq/paginator.rb +17 -9
  37. data/lib/sidekiq/processor.rb +81 -80
  38. data/lib/sidekiq/rails.rb +21 -14
  39. data/lib/sidekiq/redis_client_adapter.rb +95 -0
  40. data/lib/sidekiq/redis_connection.rb +14 -82
  41. data/lib/sidekiq/ring_buffer.rb +29 -0
  42. data/lib/sidekiq/scheduled.rb +76 -38
  43. data/lib/sidekiq/testing/inline.rb +4 -4
  44. data/lib/sidekiq/testing.rb +42 -69
  45. data/lib/sidekiq/transaction_aware_client.rb +44 -0
  46. data/lib/sidekiq/version.rb +2 -1
  47. data/lib/sidekiq/web/action.rb +3 -3
  48. data/lib/sidekiq/web/application.rb +95 -11
  49. data/lib/sidekiq/web/csrf_protection.rb +4 -4
  50. data/lib/sidekiq/web/helpers.rb +58 -30
  51. data/lib/sidekiq/web.rb +22 -17
  52. data/lib/sidekiq/worker_compatibility_alias.rb +13 -0
  53. data/lib/sidekiq.rb +85 -202
  54. data/sidekiq.gemspec +12 -10
  55. data/web/assets/javascripts/application.js +77 -26
  56. data/web/assets/javascripts/base-charts.js +106 -0
  57. data/web/assets/javascripts/chart.min.js +13 -0
  58. data/web/assets/javascripts/chartjs-plugin-annotation.min.js +7 -0
  59. data/web/assets/javascripts/dashboard-charts.js +168 -0
  60. data/web/assets/javascripts/dashboard.js +3 -240
  61. data/web/assets/javascripts/metrics.js +264 -0
  62. data/web/assets/stylesheets/application-dark.css +17 -17
  63. data/web/assets/stylesheets/application-rtl.css +2 -91
  64. data/web/assets/stylesheets/application.css +69 -302
  65. data/web/locales/ar.yml +70 -70
  66. data/web/locales/cs.yml +62 -62
  67. data/web/locales/da.yml +60 -53
  68. data/web/locales/de.yml +65 -65
  69. data/web/locales/el.yml +43 -24
  70. data/web/locales/en.yml +84 -69
  71. data/web/locales/es.yml +68 -68
  72. data/web/locales/fa.yml +65 -65
  73. data/web/locales/fr.yml +81 -67
  74. data/web/locales/gd.yml +99 -0
  75. data/web/locales/he.yml +65 -64
  76. data/web/locales/hi.yml +59 -59
  77. data/web/locales/it.yml +53 -53
  78. data/web/locales/ja.yml +73 -68
  79. data/web/locales/ko.yml +52 -52
  80. data/web/locales/lt.yml +66 -66
  81. data/web/locales/nb.yml +61 -61
  82. data/web/locales/nl.yml +52 -52
  83. data/web/locales/pl.yml +45 -45
  84. data/web/locales/pt-br.yml +83 -55
  85. data/web/locales/pt.yml +51 -51
  86. data/web/locales/ru.yml +67 -66
  87. data/web/locales/sv.yml +53 -53
  88. data/web/locales/ta.yml +60 -60
  89. data/web/locales/uk.yml +62 -61
  90. data/web/locales/ur.yml +64 -64
  91. data/web/locales/vi.yml +67 -67
  92. data/web/locales/zh-cn.yml +43 -16
  93. data/web/locales/zh-tw.yml +42 -8
  94. data/web/views/_footer.erb +5 -2
  95. data/web/views/_job_info.erb +18 -2
  96. data/web/views/_metrics_period_select.erb +12 -0
  97. data/web/views/_nav.erb +1 -1
  98. data/web/views/_paging.erb +2 -0
  99. data/web/views/_poll_link.erb +1 -1
  100. data/web/views/_summary.erb +1 -1
  101. data/web/views/busy.erb +44 -28
  102. data/web/views/dashboard.erb +36 -4
  103. data/web/views/filtering.erb +7 -0
  104. data/web/views/metrics.erb +82 -0
  105. data/web/views/metrics_for_job.erb +68 -0
  106. data/web/views/morgue.erb +5 -9
  107. data/web/views/queue.erb +15 -15
  108. data/web/views/queues.erb +3 -1
  109. data/web/views/retries.erb +5 -9
  110. data/web/views/scheduled.erb +12 -13
  111. metadata +62 -31
  112. data/LICENSE +0 -9
  113. data/lib/generators/sidekiq/worker_generator.rb +0 -57
  114. data/lib/sidekiq/delay.rb +0 -41
  115. data/lib/sidekiq/exception_handler.rb +0 -27
  116. data/lib/sidekiq/extensions/action_mailer.rb +0 -48
  117. data/lib/sidekiq/extensions/active_record.rb +0 -43
  118. data/lib/sidekiq/extensions/class_methods.rb +0 -43
  119. data/lib/sidekiq/extensions/generic_proxy.rb +0 -33
  120. data/lib/sidekiq/worker.rb +0 -311
@@ -1,311 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- require "sidekiq/client"
4
-
5
- module Sidekiq
6
- ##
7
- # Include this module in your worker class and you can easily create
8
- # asynchronous jobs:
9
- #
10
- # class HardWorker
11
- # include Sidekiq::Worker
12
- # sidekiq_options queue: 'critical', retry: 5
13
- #
14
- # def perform(*args)
15
- # # do some work
16
- # end
17
- # end
18
- #
19
- # Then in your Rails app, you can do this:
20
- #
21
- # HardWorker.perform_async(1, 2, 3)
22
- #
23
- # Note that perform_async is a class method, perform is an instance method.
24
- #
25
- # Sidekiq::Worker also includes several APIs to provide compatibility with
26
- # ActiveJob.
27
- #
28
- # class SomeWorker
29
- # include Sidekiq::Worker
30
- # queue_as :critical
31
- #
32
- # def perform(...)
33
- # end
34
- # end
35
- #
36
- # SomeWorker.set(wait_until: 1.hour).perform_async(123)
37
- #
38
- # Note that arguments passed to the job must still obey Sidekiq's
39
- # best practice for simple, JSON-native data types. Sidekiq will not
40
- # implement ActiveJob's more complex argument serialization. For
41
- # this reason, we don't implement `perform_later` as our call semantics
42
- # are very different.
43
- #
44
- module Worker
45
- ##
46
- # The Options module is extracted so we can include it in ActiveJob::Base
47
- # and allow native AJs to configure Sidekiq features/internals.
48
- module Options
49
- def self.included(base)
50
- base.extend(ClassMethods)
51
- base.sidekiq_class_attribute :sidekiq_options_hash
52
- base.sidekiq_class_attribute :sidekiq_retry_in_block
53
- base.sidekiq_class_attribute :sidekiq_retries_exhausted_block
54
- end
55
-
56
- module ClassMethods
57
- ACCESSOR_MUTEX = Mutex.new
58
-
59
- ##
60
- # Allows customization for this type of Worker.
61
- # Legal options:
62
- #
63
- # queue - name of queue to use for this job type, default *default*
64
- # retry - enable retries for this Worker in case of error during execution,
65
- # *true* to use the default or *Integer* count
66
- # backtrace - whether to save any error backtrace in the retry payload to display in web UI,
67
- # can be true, false or an integer number of lines to save, default *false*
68
- #
69
- # In practice, any option is allowed. This is the main mechanism to configure the
70
- # options for a specific job.
71
- def sidekiq_options(opts = {})
72
- opts = opts.transform_keys(&:to_s) # stringify
73
- self.sidekiq_options_hash = get_sidekiq_options.merge(opts)
74
- end
75
-
76
- def sidekiq_retry_in(&block)
77
- self.sidekiq_retry_in_block = block
78
- end
79
-
80
- def sidekiq_retries_exhausted(&block)
81
- self.sidekiq_retries_exhausted_block = block
82
- end
83
-
84
- def get_sidekiq_options # :nodoc:
85
- self.sidekiq_options_hash ||= Sidekiq.default_worker_options
86
- end
87
-
88
- def sidekiq_class_attribute(*attrs)
89
- instance_reader = true
90
- instance_writer = true
91
-
92
- attrs.each do |name|
93
- synchronized_getter = "__synchronized_#{name}"
94
-
95
- singleton_class.instance_eval do
96
- undef_method(name) if method_defined?(name) || private_method_defined?(name)
97
- end
98
-
99
- define_singleton_method(synchronized_getter) { nil }
100
- singleton_class.class_eval do
101
- private(synchronized_getter)
102
- end
103
-
104
- define_singleton_method(name) { ACCESSOR_MUTEX.synchronize { send synchronized_getter } }
105
-
106
- ivar = "@#{name}"
107
-
108
- singleton_class.instance_eval do
109
- m = "#{name}="
110
- undef_method(m) if method_defined?(m) || private_method_defined?(m)
111
- end
112
- define_singleton_method("#{name}=") do |val|
113
- singleton_class.class_eval do
114
- ACCESSOR_MUTEX.synchronize do
115
- undef_method(synchronized_getter) if method_defined?(synchronized_getter) || private_method_defined?(synchronized_getter)
116
- define_method(synchronized_getter) { val }
117
- end
118
- end
119
-
120
- if singleton_class?
121
- class_eval do
122
- undef_method(name) if method_defined?(name) || private_method_defined?(name)
123
- define_method(name) do
124
- if instance_variable_defined? ivar
125
- instance_variable_get ivar
126
- else
127
- singleton_class.send name
128
- end
129
- end
130
- end
131
- end
132
- val
133
- end
134
-
135
- if instance_reader
136
- undef_method(name) if method_defined?(name) || private_method_defined?(name)
137
- define_method(name) do
138
- if instance_variable_defined?(ivar)
139
- instance_variable_get ivar
140
- else
141
- self.class.public_send name
142
- end
143
- end
144
- end
145
-
146
- if instance_writer
147
- m = "#{name}="
148
- undef_method(m) if method_defined?(m) || private_method_defined?(m)
149
- attr_writer name
150
- end
151
- end
152
- end
153
- end
154
- end
155
-
156
- attr_accessor :jid
157
-
158
- def self.included(base)
159
- raise ArgumentError, "Sidekiq::Worker cannot be included in an ActiveJob: #{base.name}" if base.ancestors.any? { |c| c.name == "ActiveJob::Base" }
160
-
161
- base.include(Options)
162
- base.extend(ClassMethods)
163
- end
164
-
165
- def logger
166
- Sidekiq.logger
167
- end
168
-
169
- # This helper class encapsulates the set options for `set`, e.g.
170
- #
171
- # SomeWorker.set(queue: 'foo').perform_async(....)
172
- #
173
- class Setter
174
- def initialize(klass, opts)
175
- @klass = klass
176
- @opts = opts
177
-
178
- # ActiveJob compatibility
179
- interval = @opts.delete(:wait_until) || @opts.delete(:wait)
180
- at(interval) if interval
181
- end
182
-
183
- def set(options)
184
- interval = options.delete(:wait_until) || options.delete(:wait)
185
- @opts.merge!(options)
186
- at(interval) if interval
187
- self
188
- end
189
-
190
- def perform_async(*args)
191
- @klass.client_push(@opts.merge("args" => args, "class" => @klass))
192
- end
193
-
194
- def perform_bulk(args, batch_size: 1_000)
195
- args.each_slice(batch_size).flat_map do |slice|
196
- Sidekiq::Client.push_bulk(@opts.merge("class" => @klass, "args" => slice))
197
- end
198
- end
199
-
200
- # +interval+ must be a timestamp, numeric or something that acts
201
- # numeric (like an activesupport time interval).
202
- def perform_in(interval, *args)
203
- at(interval).perform_async(*args)
204
- end
205
- alias_method :perform_at, :perform_in
206
-
207
- private
208
-
209
- def at(interval)
210
- int = interval.to_f
211
- now = Time.now.to_f
212
- ts = (int < 1_000_000_000 ? now + int : int)
213
- # Optimization to enqueue something now that is scheduled to go out now or in the past
214
- @opts["at"] = ts if ts > now
215
- self
216
- end
217
- end
218
-
219
- module ClassMethods
220
- def delay(*args)
221
- raise ArgumentError, "Do not call .delay on a Sidekiq::Worker class, call .perform_async"
222
- end
223
-
224
- def delay_for(*args)
225
- raise ArgumentError, "Do not call .delay_for on a Sidekiq::Worker class, call .perform_in"
226
- end
227
-
228
- def delay_until(*args)
229
- raise ArgumentError, "Do not call .delay_until on a Sidekiq::Worker class, call .perform_at"
230
- end
231
-
232
- def queue_as(q)
233
- sidekiq_options("queue" => q.to_s)
234
- end
235
-
236
- def set(options)
237
- Setter.new(self, options)
238
- end
239
-
240
- def perform_async(*args)
241
- client_push("class" => self, "args" => args)
242
- end
243
-
244
- ##
245
- # Push a large number of jobs to Redis, while limiting the batch of
246
- # each job payload to 1,000. This method helps cut down on the number
247
- # of round trips to Redis, which can increase the performance of enqueueing
248
- # large numbers of jobs.
249
- #
250
- # +items+ must be an Array of Arrays.
251
- #
252
- # For finer-grained control, use `Sidekiq::Client.push_bulk` directly.
253
- #
254
- # Example (3 Redis round trips):
255
- #
256
- # SomeWorker.perform_async(1)
257
- # SomeWorker.perform_async(2)
258
- # SomeWorker.perform_async(3)
259
- #
260
- # Would instead become (1 Redis round trip):
261
- #
262
- # SomeWorker.perform_bulk([[1], [2], [3]])
263
- #
264
- def perform_bulk(items, batch_size: 1_000)
265
- items.each_slice(batch_size).flat_map do |slice|
266
- Sidekiq::Client.push_bulk("class" => self, "args" => slice)
267
- end
268
- end
269
-
270
- # +interval+ must be a timestamp, numeric or something that acts
271
- # numeric (like an activesupport time interval).
272
- def perform_in(interval, *args)
273
- int = interval.to_f
274
- now = Time.now.to_f
275
- ts = (int < 1_000_000_000 ? now + int : int)
276
-
277
- item = {"class" => self, "args" => args}
278
-
279
- # Optimization to enqueue something now that is scheduled to go out now or in the past
280
- item["at"] = ts if ts > now
281
-
282
- client_push(item)
283
- end
284
- alias_method :perform_at, :perform_in
285
-
286
- ##
287
- # Allows customization for this type of Worker.
288
- # Legal options:
289
- #
290
- # queue - use a named queue for this Worker, default 'default'
291
- # retry - enable the RetryJobs middleware for this Worker, *true* to use the default
292
- # or *Integer* count
293
- # backtrace - whether to save any error backtrace in the retry payload to display in web UI,
294
- # can be true, false or an integer number of lines to save, default *false*
295
- # pool - use the given Redis connection pool to push this type of job to a given shard.
296
- #
297
- # In practice, any option is allowed. This is the main mechanism to configure the
298
- # options for a specific job.
299
- def sidekiq_options(opts = {})
300
- super
301
- end
302
-
303
- def client_push(item) # :nodoc:
304
- pool = Thread.current[:sidekiq_via_pool] || get_sidekiq_options["pool"] || Sidekiq.redis_pool
305
- stringified_item = item.transform_keys(&:to_s)
306
-
307
- Sidekiq::Client.new(pool).push(stringified_item)
308
- end
309
- end
310
- end
311
- end