sidekiq 6.1.0

Sign up to get free protection for your applications and to get access to all the features.

Potentially problematic release.


This version of sidekiq might be problematic. Click here for more details.

Files changed (127) hide show
  1. checksums.yaml +7 -0
  2. data/.circleci/config.yml +71 -0
  3. data/.github/contributing.md +32 -0
  4. data/.github/issue_template.md +11 -0
  5. data/.gitignore +13 -0
  6. data/.standard.yml +20 -0
  7. data/3.0-Upgrade.md +70 -0
  8. data/4.0-Upgrade.md +53 -0
  9. data/5.0-Upgrade.md +56 -0
  10. data/6.0-Upgrade.md +72 -0
  11. data/COMM-LICENSE +97 -0
  12. data/Changes.md +1718 -0
  13. data/Ent-2.0-Upgrade.md +37 -0
  14. data/Ent-Changes.md +269 -0
  15. data/Gemfile +24 -0
  16. data/Gemfile.lock +208 -0
  17. data/LICENSE +9 -0
  18. data/Pro-2.0-Upgrade.md +138 -0
  19. data/Pro-3.0-Upgrade.md +44 -0
  20. data/Pro-4.0-Upgrade.md +35 -0
  21. data/Pro-5.0-Upgrade.md +25 -0
  22. data/Pro-Changes.md +790 -0
  23. data/README.md +94 -0
  24. data/Rakefile +10 -0
  25. data/bin/sidekiq +42 -0
  26. data/bin/sidekiqload +157 -0
  27. data/bin/sidekiqmon +8 -0
  28. data/code_of_conduct.md +50 -0
  29. data/lib/generators/sidekiq/templates/worker.rb.erb +9 -0
  30. data/lib/generators/sidekiq/templates/worker_spec.rb.erb +6 -0
  31. data/lib/generators/sidekiq/templates/worker_test.rb.erb +8 -0
  32. data/lib/generators/sidekiq/worker_generator.rb +57 -0
  33. data/lib/sidekiq.rb +262 -0
  34. data/lib/sidekiq/api.rb +960 -0
  35. data/lib/sidekiq/cli.rb +401 -0
  36. data/lib/sidekiq/client.rb +263 -0
  37. data/lib/sidekiq/delay.rb +41 -0
  38. data/lib/sidekiq/exception_handler.rb +27 -0
  39. data/lib/sidekiq/extensions/action_mailer.rb +47 -0
  40. data/lib/sidekiq/extensions/active_record.rb +43 -0
  41. data/lib/sidekiq/extensions/class_methods.rb +43 -0
  42. data/lib/sidekiq/extensions/generic_proxy.rb +31 -0
  43. data/lib/sidekiq/fetch.rb +82 -0
  44. data/lib/sidekiq/job_logger.rb +63 -0
  45. data/lib/sidekiq/job_retry.rb +262 -0
  46. data/lib/sidekiq/launcher.rb +206 -0
  47. data/lib/sidekiq/logger.rb +165 -0
  48. data/lib/sidekiq/manager.rb +135 -0
  49. data/lib/sidekiq/middleware/chain.rb +160 -0
  50. data/lib/sidekiq/middleware/i18n.rb +40 -0
  51. data/lib/sidekiq/monitor.rb +133 -0
  52. data/lib/sidekiq/paginator.rb +47 -0
  53. data/lib/sidekiq/processor.rb +280 -0
  54. data/lib/sidekiq/rails.rb +50 -0
  55. data/lib/sidekiq/redis_connection.rb +146 -0
  56. data/lib/sidekiq/scheduled.rb +173 -0
  57. data/lib/sidekiq/sd_notify.rb +149 -0
  58. data/lib/sidekiq/systemd.rb +24 -0
  59. data/lib/sidekiq/testing.rb +344 -0
  60. data/lib/sidekiq/testing/inline.rb +30 -0
  61. data/lib/sidekiq/util.rb +67 -0
  62. data/lib/sidekiq/version.rb +5 -0
  63. data/lib/sidekiq/web.rb +213 -0
  64. data/lib/sidekiq/web/action.rb +93 -0
  65. data/lib/sidekiq/web/application.rb +357 -0
  66. data/lib/sidekiq/web/csrf_protection.rb +153 -0
  67. data/lib/sidekiq/web/helpers.rb +333 -0
  68. data/lib/sidekiq/web/router.rb +101 -0
  69. data/lib/sidekiq/worker.rb +244 -0
  70. data/sidekiq.gemspec +20 -0
  71. data/web/assets/images/favicon.ico +0 -0
  72. data/web/assets/images/logo.png +0 -0
  73. data/web/assets/images/status.png +0 -0
  74. data/web/assets/javascripts/application.js +95 -0
  75. data/web/assets/javascripts/dashboard.js +296 -0
  76. data/web/assets/stylesheets/application-dark.css +133 -0
  77. data/web/assets/stylesheets/application-rtl.css +246 -0
  78. data/web/assets/stylesheets/application.css +1158 -0
  79. data/web/assets/stylesheets/bootstrap-rtl.min.css +9 -0
  80. data/web/assets/stylesheets/bootstrap.css +5 -0
  81. data/web/locales/ar.yml +81 -0
  82. data/web/locales/cs.yml +78 -0
  83. data/web/locales/da.yml +68 -0
  84. data/web/locales/de.yml +81 -0
  85. data/web/locales/el.yml +68 -0
  86. data/web/locales/en.yml +83 -0
  87. data/web/locales/es.yml +70 -0
  88. data/web/locales/fa.yml +80 -0
  89. data/web/locales/fr.yml +78 -0
  90. data/web/locales/he.yml +79 -0
  91. data/web/locales/hi.yml +75 -0
  92. data/web/locales/it.yml +69 -0
  93. data/web/locales/ja.yml +83 -0
  94. data/web/locales/ko.yml +68 -0
  95. data/web/locales/lt.yml +83 -0
  96. data/web/locales/nb.yml +77 -0
  97. data/web/locales/nl.yml +68 -0
  98. data/web/locales/pl.yml +59 -0
  99. data/web/locales/pt-br.yml +68 -0
  100. data/web/locales/pt.yml +67 -0
  101. data/web/locales/ru.yml +78 -0
  102. data/web/locales/sv.yml +68 -0
  103. data/web/locales/ta.yml +75 -0
  104. data/web/locales/uk.yml +76 -0
  105. data/web/locales/ur.yml +80 -0
  106. data/web/locales/vi.yml +83 -0
  107. data/web/locales/zh-cn.yml +68 -0
  108. data/web/locales/zh-tw.yml +68 -0
  109. data/web/views/_footer.erb +20 -0
  110. data/web/views/_job_info.erb +89 -0
  111. data/web/views/_nav.erb +52 -0
  112. data/web/views/_paging.erb +23 -0
  113. data/web/views/_poll_link.erb +7 -0
  114. data/web/views/_status.erb +4 -0
  115. data/web/views/_summary.erb +40 -0
  116. data/web/views/busy.erb +101 -0
  117. data/web/views/dashboard.erb +75 -0
  118. data/web/views/dead.erb +34 -0
  119. data/web/views/layout.erb +41 -0
  120. data/web/views/morgue.erb +78 -0
  121. data/web/views/queue.erb +55 -0
  122. data/web/views/queues.erb +38 -0
  123. data/web/views/retries.erb +83 -0
  124. data/web/views/retry.erb +34 -0
  125. data/web/views/scheduled.erb +57 -0
  126. data/web/views/scheduled_job_info.erb +8 -0
  127. metadata +212 -0
@@ -0,0 +1,24 @@
1
+ #
2
+ # Sidekiq's systemd integration allows Sidekiq to inform systemd:
3
+ # 1. when it has successfully started
4
+ # 2. when it is starting shutdown
5
+ # 3. periodically for a liveness check with a watchdog thread
6
+ #
7
+ module Sidekiq
8
+ def self.start_watchdog
9
+ usec = Integer(ENV["WATCHDOG_USEC"])
10
+ return Sidekiq.logger.error("systemd Watchdog too fast: " + usec) if usec < 1_000_000
11
+
12
+ sec_f = usec / 1_000_000.0
13
+ # "It is recommended that a daemon sends a keep-alive notification message
14
+ # to the service manager every half of the time returned here."
15
+ ping_f = sec_f / 2
16
+ Sidekiq.logger.info "Pinging systemd watchdog every #{ping_f.round(1)} sec"
17
+ Thread.new do
18
+ loop do
19
+ sleep ping_f
20
+ Sidekiq::SdNotify.watchdog
21
+ end
22
+ end
23
+ end
24
+ end
@@ -0,0 +1,344 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "securerandom"
4
+ require "sidekiq"
5
+
6
+ module Sidekiq
7
+ class Testing
8
+ class << self
9
+ attr_accessor :__test_mode
10
+
11
+ def __set_test_mode(mode)
12
+ if block_given?
13
+ current_mode = __test_mode
14
+ begin
15
+ self.__test_mode = mode
16
+ yield
17
+ ensure
18
+ self.__test_mode = current_mode
19
+ end
20
+ else
21
+ self.__test_mode = mode
22
+ end
23
+ end
24
+
25
+ def disable!(&block)
26
+ __set_test_mode(:disable, &block)
27
+ end
28
+
29
+ def fake!(&block)
30
+ __set_test_mode(:fake, &block)
31
+ end
32
+
33
+ def inline!(&block)
34
+ __set_test_mode(:inline, &block)
35
+ end
36
+
37
+ def enabled?
38
+ __test_mode != :disable
39
+ end
40
+
41
+ def disabled?
42
+ __test_mode == :disable
43
+ end
44
+
45
+ def fake?
46
+ __test_mode == :fake
47
+ end
48
+
49
+ def inline?
50
+ __test_mode == :inline
51
+ end
52
+
53
+ def server_middleware
54
+ @server_chain ||= Middleware::Chain.new
55
+ yield @server_chain if block_given?
56
+ @server_chain
57
+ end
58
+
59
+ def constantize(str)
60
+ names = str.split("::")
61
+ names.shift if names.empty? || names.first.empty?
62
+
63
+ names.inject(Object) do |constant, name|
64
+ constant.const_defined?(name) ? constant.const_get(name) : constant.const_missing(name)
65
+ end
66
+ end
67
+ end
68
+ end
69
+
70
+ # Default to fake testing to keep old behavior
71
+ Sidekiq::Testing.fake!
72
+
73
+ class EmptyQueueError < RuntimeError; end
74
+
75
+ module TestingClient
76
+ def raw_push(payloads)
77
+ if Sidekiq::Testing.fake?
78
+ payloads.each do |job|
79
+ job = Sidekiq.load_json(Sidekiq.dump_json(job))
80
+ job["enqueued_at"] = Time.now.to_f unless job["at"]
81
+ Queues.push(job["queue"], job["class"], job)
82
+ end
83
+ true
84
+ elsif Sidekiq::Testing.inline?
85
+ payloads.each do |job|
86
+ klass = Sidekiq::Testing.constantize(job["class"])
87
+ job["id"] ||= SecureRandom.hex(12)
88
+ job_hash = Sidekiq.load_json(Sidekiq.dump_json(job))
89
+ klass.process_job(job_hash)
90
+ end
91
+ true
92
+ else
93
+ super
94
+ end
95
+ end
96
+ end
97
+
98
+ Sidekiq::Client.prepend TestingClient
99
+
100
+ module Queues
101
+ ##
102
+ # The Queues class is only for testing the fake queue implementation.
103
+ # There are 2 data structures involved in tandem. This is due to the
104
+ # Rspec syntax of change(QueueWorker.jobs, :size). It keeps a reference
105
+ # to the array. Because the array was dervied from a filter of the total
106
+ # jobs enqueued, it appeared as though the array didn't change.
107
+ #
108
+ # To solve this, we'll keep 2 hashes containing the jobs. One with keys based
109
+ # on the queue, and another with keys of the worker names, so the array for
110
+ # QueueWorker.jobs is a straight reference to a real array.
111
+ #
112
+ # Queue-based hash:
113
+ #
114
+ # {
115
+ # "default"=>[
116
+ # {
117
+ # "class"=>"TestTesting::QueueWorker",
118
+ # "args"=>[1, 2],
119
+ # "retry"=>true,
120
+ # "queue"=>"default",
121
+ # "jid"=>"abc5b065c5c4b27fc1102833",
122
+ # "created_at"=>1447445554.419934
123
+ # }
124
+ # ]
125
+ # }
126
+ #
127
+ # Worker-based hash:
128
+ #
129
+ # {
130
+ # "TestTesting::QueueWorker"=>[
131
+ # {
132
+ # "class"=>"TestTesting::QueueWorker",
133
+ # "args"=>[1, 2],
134
+ # "retry"=>true,
135
+ # "queue"=>"default",
136
+ # "jid"=>"abc5b065c5c4b27fc1102833",
137
+ # "created_at"=>1447445554.419934
138
+ # }
139
+ # ]
140
+ # }
141
+ #
142
+ # Example:
143
+ #
144
+ # require 'sidekiq/testing'
145
+ #
146
+ # assert_equal 0, Sidekiq::Queues["default"].size
147
+ # HardWorker.perform_async(:something)
148
+ # assert_equal 1, Sidekiq::Queues["default"].size
149
+ # assert_equal :something, Sidekiq::Queues["default"].first['args'][0]
150
+ #
151
+ # You can also clear all workers' jobs:
152
+ #
153
+ # assert_equal 0, Sidekiq::Queues["default"].size
154
+ # HardWorker.perform_async(:something)
155
+ # Sidekiq::Queues.clear_all
156
+ # assert_equal 0, Sidekiq::Queues["default"].size
157
+ #
158
+ # This can be useful to make sure jobs don't linger between tests:
159
+ #
160
+ # RSpec.configure do |config|
161
+ # config.before(:each) do
162
+ # Sidekiq::Queues.clear_all
163
+ # end
164
+ # end
165
+ #
166
+ class << self
167
+ def [](queue)
168
+ jobs_by_queue[queue]
169
+ end
170
+
171
+ def push(queue, klass, job)
172
+ jobs_by_queue[queue] << job
173
+ jobs_by_worker[klass] << job
174
+ end
175
+
176
+ def jobs_by_queue
177
+ @jobs_by_queue ||= Hash.new { |hash, key| hash[key] = [] }
178
+ end
179
+
180
+ def jobs_by_worker
181
+ @jobs_by_worker ||= Hash.new { |hash, key| hash[key] = [] }
182
+ end
183
+
184
+ def delete_for(jid, queue, klass)
185
+ jobs_by_queue[queue.to_s].delete_if { |job| job["jid"] == jid }
186
+ jobs_by_worker[klass].delete_if { |job| job["jid"] == jid }
187
+ end
188
+
189
+ def clear_for(queue, klass)
190
+ jobs_by_queue[queue].clear
191
+ jobs_by_worker[klass].clear
192
+ end
193
+
194
+ def clear_all
195
+ jobs_by_queue.clear
196
+ jobs_by_worker.clear
197
+ end
198
+ end
199
+ end
200
+
201
+ module Worker
202
+ ##
203
+ # The Sidekiq testing infrastructure overrides perform_async
204
+ # so that it does not actually touch the network. Instead it
205
+ # stores the asynchronous jobs in a per-class array so that
206
+ # their presence/absence can be asserted by your tests.
207
+ #
208
+ # This is similar to ActionMailer's :test delivery_method and its
209
+ # ActionMailer::Base.deliveries array.
210
+ #
211
+ # Example:
212
+ #
213
+ # require 'sidekiq/testing'
214
+ #
215
+ # assert_equal 0, HardWorker.jobs.size
216
+ # HardWorker.perform_async(:something)
217
+ # assert_equal 1, HardWorker.jobs.size
218
+ # assert_equal :something, HardWorker.jobs[0]['args'][0]
219
+ #
220
+ # assert_equal 0, Sidekiq::Extensions::DelayedMailer.jobs.size
221
+ # MyMailer.delay.send_welcome_email('foo@example.com')
222
+ # assert_equal 1, Sidekiq::Extensions::DelayedMailer.jobs.size
223
+ #
224
+ # You can also clear and drain all workers' jobs:
225
+ #
226
+ # assert_equal 0, Sidekiq::Extensions::DelayedMailer.jobs.size
227
+ # assert_equal 0, Sidekiq::Extensions::DelayedModel.jobs.size
228
+ #
229
+ # MyMailer.delay.send_welcome_email('foo@example.com')
230
+ # MyModel.delay.do_something_hard
231
+ #
232
+ # assert_equal 1, Sidekiq::Extensions::DelayedMailer.jobs.size
233
+ # assert_equal 1, Sidekiq::Extensions::DelayedModel.jobs.size
234
+ #
235
+ # Sidekiq::Worker.clear_all # or .drain_all
236
+ #
237
+ # assert_equal 0, Sidekiq::Extensions::DelayedMailer.jobs.size
238
+ # assert_equal 0, Sidekiq::Extensions::DelayedModel.jobs.size
239
+ #
240
+ # This can be useful to make sure jobs don't linger between tests:
241
+ #
242
+ # RSpec.configure do |config|
243
+ # config.before(:each) do
244
+ # Sidekiq::Worker.clear_all
245
+ # end
246
+ # end
247
+ #
248
+ # or for acceptance testing, i.e. with cucumber:
249
+ #
250
+ # AfterStep do
251
+ # Sidekiq::Worker.drain_all
252
+ # end
253
+ #
254
+ # When I sign up as "foo@example.com"
255
+ # Then I should receive a welcome email to "foo@example.com"
256
+ #
257
+ module ClassMethods
258
+ # Queue for this worker
259
+ def queue
260
+ get_sidekiq_options["queue"]
261
+ end
262
+
263
+ # Jobs queued for this worker
264
+ def jobs
265
+ Queues.jobs_by_worker[to_s]
266
+ end
267
+
268
+ # Clear all jobs for this worker
269
+ def clear
270
+ Queues.clear_for(queue, to_s)
271
+ end
272
+
273
+ # Drain and run all jobs for this worker
274
+ def drain
275
+ while jobs.any?
276
+ next_job = jobs.first
277
+ Queues.delete_for(next_job["jid"], next_job["queue"], to_s)
278
+ process_job(next_job)
279
+ end
280
+ end
281
+
282
+ # Pop out a single job and perform it
283
+ def perform_one
284
+ raise(EmptyQueueError, "perform_one called with empty job queue") if jobs.empty?
285
+ next_job = jobs.first
286
+ Queues.delete_for(next_job["jid"], queue, to_s)
287
+ process_job(next_job)
288
+ end
289
+
290
+ def process_job(job)
291
+ worker = new
292
+ worker.jid = job["jid"]
293
+ worker.bid = job["bid"] if worker.respond_to?(:bid=)
294
+ Sidekiq::Testing.server_middleware.invoke(worker, job, job["queue"]) do
295
+ execute_job(worker, job["args"])
296
+ end
297
+ end
298
+
299
+ def execute_job(worker, args)
300
+ worker.perform(*args)
301
+ end
302
+ end
303
+
304
+ class << self
305
+ def jobs # :nodoc:
306
+ Queues.jobs_by_queue.values.flatten
307
+ end
308
+
309
+ # Clear all queued jobs across all workers
310
+ def clear_all
311
+ Queues.clear_all
312
+ end
313
+
314
+ # Drain all queued jobs across all workers
315
+ def drain_all
316
+ while jobs.any?
317
+ worker_classes = jobs.map { |job| job["class"] }.uniq
318
+
319
+ worker_classes.each do |worker_class|
320
+ Sidekiq::Testing.constantize(worker_class).drain
321
+ end
322
+ end
323
+ end
324
+ end
325
+ end
326
+
327
+ module TestingExtensions
328
+ def jobs_for(klass)
329
+ jobs.select do |job|
330
+ marshalled = job["args"][0]
331
+ marshalled.index(klass.to_s) && YAML.load(marshalled)[0] == klass
332
+ end
333
+ end
334
+ end
335
+
336
+ Sidekiq::Extensions::DelayedMailer.extend(TestingExtensions) if defined?(Sidekiq::Extensions::DelayedMailer)
337
+ Sidekiq::Extensions::DelayedModel.extend(TestingExtensions) if defined?(Sidekiq::Extensions::DelayedModel)
338
+ end
339
+
340
+ if defined?(::Rails) && Rails.respond_to?(:env) && !Rails.env.test? && !$TESTING
341
+ puts("**************************************************")
342
+ puts("⛔️ WARNING: Sidekiq testing API enabled, but this is not the test environment. Your jobs will not go to Redis.")
343
+ puts("**************************************************")
344
+ end
@@ -0,0 +1,30 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "sidekiq/testing"
4
+
5
+ ##
6
+ # The Sidekiq inline infrastructure overrides perform_async so that it
7
+ # actually calls perform instead. This allows workers to be run inline in a
8
+ # testing environment.
9
+ #
10
+ # This is similar to `Resque.inline = true` functionality.
11
+ #
12
+ # Example:
13
+ #
14
+ # require 'sidekiq/testing/inline'
15
+ #
16
+ # $external_variable = 0
17
+ #
18
+ # class ExternalWorker
19
+ # include Sidekiq::Worker
20
+ #
21
+ # def perform
22
+ # $external_variable = 1
23
+ # end
24
+ # end
25
+ #
26
+ # assert_equal 0, $external_variable
27
+ # ExternalWorker.perform_async
28
+ # assert_equal 1, $external_variable
29
+ #
30
+ Sidekiq::Testing.inline!
@@ -0,0 +1,67 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "socket"
4
+ require "securerandom"
5
+ require "sidekiq/exception_handler"
6
+
7
+ module Sidekiq
8
+ ##
9
+ # This module is part of Sidekiq core and not intended for extensions.
10
+ #
11
+ module Util
12
+ include ExceptionHandler
13
+
14
+ def watchdog(last_words)
15
+ yield
16
+ rescue Exception => ex
17
+ handle_exception(ex, {context: last_words})
18
+ raise ex
19
+ end
20
+
21
+ def safe_thread(name, &block)
22
+ Thread.new do
23
+ Thread.current.name = name
24
+ watchdog(name, &block)
25
+ end
26
+ end
27
+
28
+ def logger
29
+ Sidekiq.logger
30
+ end
31
+
32
+ def redis(&block)
33
+ Sidekiq.redis(&block)
34
+ end
35
+
36
+ def tid
37
+ Thread.current["sidekiq_tid"] ||= (Thread.current.object_id ^ ::Process.pid).to_s(36)
38
+ end
39
+
40
+ def hostname
41
+ ENV["DYNO"] || Socket.gethostname
42
+ end
43
+
44
+ def process_nonce
45
+ @@process_nonce ||= SecureRandom.hex(6)
46
+ end
47
+
48
+ def identity
49
+ @@identity ||= "#{hostname}:#{::Process.pid}:#{process_nonce}"
50
+ end
51
+
52
+ def fire_event(event, options = {})
53
+ reverse = options[:reverse]
54
+ reraise = options[:reraise]
55
+
56
+ arr = Sidekiq.options[:lifecycle_events][event]
57
+ arr.reverse! if reverse
58
+ arr.each do |block|
59
+ block.call
60
+ rescue => ex
61
+ handle_exception(ex, {context: "Exception during Sidekiq lifecycle event.", event: event})
62
+ raise ex if reraise
63
+ end
64
+ arr.clear
65
+ end
66
+ end
67
+ end