sidekiq 4.2.7 → 5.2.8

Sign up to get free protection for your applications and to get access to all the features.

Potentially problematic release.


This version of sidekiq might be problematic. Click here for more details.

Files changed (108) hide show
  1. checksums.yaml +5 -5
  2. data/.circleci/config.yml +61 -0
  3. data/.github/issue_template.md +8 -1
  4. data/.gitignore +3 -0
  5. data/.travis.yml +5 -6
  6. data/5.0-Upgrade.md +56 -0
  7. data/COMM-LICENSE +12 -10
  8. data/Changes.md +187 -0
  9. data/Ent-Changes.md +84 -3
  10. data/Gemfile +14 -20
  11. data/LICENSE +1 -1
  12. data/Pro-4.0-Upgrade.md +35 -0
  13. data/Pro-Changes.md +172 -3
  14. data/README.md +8 -6
  15. data/Rakefile +2 -5
  16. data/bin/sidekiqctl +13 -92
  17. data/bin/sidekiqload +16 -21
  18. data/lib/generators/sidekiq/templates/worker_spec.rb.erb +1 -1
  19. data/lib/generators/sidekiq/templates/worker_test.rb.erb +1 -1
  20. data/lib/sidekiq/api.rb +163 -67
  21. data/lib/sidekiq/cli.rb +121 -78
  22. data/lib/sidekiq/client.rb +25 -18
  23. data/lib/sidekiq/core_ext.rb +1 -119
  24. data/lib/sidekiq/ctl.rb +221 -0
  25. data/lib/sidekiq/delay.rb +42 -0
  26. data/lib/sidekiq/exception_handler.rb +2 -4
  27. data/lib/sidekiq/extensions/generic_proxy.rb +7 -1
  28. data/lib/sidekiq/fetch.rb +1 -1
  29. data/lib/sidekiq/job_logger.rb +25 -0
  30. data/lib/sidekiq/job_retry.rb +262 -0
  31. data/lib/sidekiq/launcher.rb +19 -19
  32. data/lib/sidekiq/logging.rb +18 -2
  33. data/lib/sidekiq/manager.rb +6 -7
  34. data/lib/sidekiq/middleware/server/active_record.rb +10 -0
  35. data/lib/sidekiq/processor.rb +126 -39
  36. data/lib/sidekiq/rails.rb +16 -77
  37. data/lib/sidekiq/redis_connection.rb +50 -5
  38. data/lib/sidekiq/scheduled.rb +35 -8
  39. data/lib/sidekiq/testing.rb +21 -6
  40. data/lib/sidekiq/util.rb +6 -2
  41. data/lib/sidekiq/version.rb +1 -1
  42. data/lib/sidekiq/web/action.rb +2 -6
  43. data/lib/sidekiq/web/application.rb +34 -17
  44. data/lib/sidekiq/web/helpers.rb +72 -23
  45. data/lib/sidekiq/web/router.rb +10 -10
  46. data/lib/sidekiq/web.rb +4 -4
  47. data/lib/sidekiq/worker.rb +118 -19
  48. data/lib/sidekiq.rb +27 -27
  49. data/sidekiq.gemspec +8 -13
  50. data/web/assets/javascripts/application.js +0 -0
  51. data/web/assets/javascripts/dashboard.js +33 -18
  52. data/web/assets/stylesheets/application-rtl.css +246 -0
  53. data/web/assets/stylesheets/application.css +371 -6
  54. data/web/assets/stylesheets/bootstrap-rtl.min.css +9 -0
  55. data/web/assets/stylesheets/bootstrap.css +2 -2
  56. data/web/locales/ar.yml +81 -0
  57. data/web/locales/en.yml +2 -0
  58. data/web/locales/es.yml +4 -3
  59. data/web/locales/fa.yml +80 -0
  60. data/web/locales/he.yml +79 -0
  61. data/web/locales/ja.yml +5 -3
  62. data/web/locales/ur.yml +80 -0
  63. data/web/views/_footer.erb +5 -2
  64. data/web/views/_job_info.erb +1 -1
  65. data/web/views/_nav.erb +4 -18
  66. data/web/views/_paging.erb +1 -1
  67. data/web/views/busy.erb +9 -5
  68. data/web/views/dashboard.erb +3 -3
  69. data/web/views/layout.erb +11 -2
  70. data/web/views/morgue.erb +6 -4
  71. data/web/views/queue.erb +11 -10
  72. data/web/views/queues.erb +4 -2
  73. data/web/views/retries.erb +9 -5
  74. data/web/views/retry.erb +1 -1
  75. data/web/views/scheduled.erb +2 -2
  76. metadata +33 -151
  77. data/lib/sidekiq/middleware/server/logging.rb +0 -40
  78. data/lib/sidekiq/middleware/server/retry_jobs.rb +0 -205
  79. data/test/config.yml +0 -9
  80. data/test/env_based_config.yml +0 -11
  81. data/test/fake_env.rb +0 -1
  82. data/test/fixtures/en.yml +0 -2
  83. data/test/helper.rb +0 -75
  84. data/test/test_actors.rb +0 -138
  85. data/test/test_api.rb +0 -528
  86. data/test/test_cli.rb +0 -418
  87. data/test/test_client.rb +0 -266
  88. data/test/test_exception_handler.rb +0 -56
  89. data/test/test_extensions.rb +0 -129
  90. data/test/test_fetch.rb +0 -50
  91. data/test/test_launcher.rb +0 -92
  92. data/test/test_logging.rb +0 -35
  93. data/test/test_manager.rb +0 -50
  94. data/test/test_middleware.rb +0 -158
  95. data/test/test_processor.rb +0 -249
  96. data/test/test_rails.rb +0 -22
  97. data/test/test_redis_connection.rb +0 -132
  98. data/test/test_retry.rb +0 -326
  99. data/test/test_retry_exhausted.rb +0 -149
  100. data/test/test_scheduled.rb +0 -115
  101. data/test/test_scheduling.rb +0 -58
  102. data/test/test_sidekiq.rb +0 -107
  103. data/test/test_testing.rb +0 -143
  104. data/test/test_testing_fake.rb +0 -359
  105. data/test/test_testing_inline.rb +0 -94
  106. data/test/test_util.rb +0 -13
  107. data/test/test_web.rb +0 -679
  108. data/test/test_web_helpers.rb +0 -54
data/lib/sidekiq/api.rb CHANGED
@@ -1,9 +1,24 @@
1
- # encoding: utf-8
2
1
  # frozen_string_literal: true
3
2
  require 'sidekiq'
4
3
 
5
4
  module Sidekiq
5
+
6
+ module RedisScanner
7
+ def sscan(conn, key)
8
+ cursor = '0'
9
+ result = []
10
+ loop do
11
+ cursor, values = conn.sscan(key, cursor)
12
+ result.push(*values)
13
+ break if cursor == '0'
14
+ end
15
+ result
16
+ end
17
+ end
18
+
6
19
  class Stats
20
+ include RedisScanner
21
+
7
22
  def initialize
8
23
  fetch_stats!
9
24
  end
@@ -51,31 +66,40 @@ module Sidekiq
51
66
  def fetch_stats!
52
67
  pipe1_res = Sidekiq.redis do |conn|
53
68
  conn.pipelined do
54
- conn.get('stat:processed'.freeze)
55
- conn.get('stat:failed'.freeze)
56
- conn.zcard('schedule'.freeze)
57
- conn.zcard('retry'.freeze)
58
- conn.zcard('dead'.freeze)
59
- conn.scard('processes'.freeze)
60
- conn.lrange('queue:default'.freeze, -1, -1)
61
- conn.smembers('processes'.freeze)
62
- conn.smembers('queues'.freeze)
69
+ conn.get('stat:processed')
70
+ conn.get('stat:failed')
71
+ conn.zcard('schedule')
72
+ conn.zcard('retry')
73
+ conn.zcard('dead')
74
+ conn.scard('processes')
75
+ conn.lrange('queue:default', -1, -1)
63
76
  end
64
77
  end
65
78
 
79
+ processes = Sidekiq.redis do |conn|
80
+ sscan(conn, 'processes')
81
+ end
82
+
83
+ queues = Sidekiq.redis do |conn|
84
+ sscan(conn, 'queues')
85
+ end
86
+
66
87
  pipe2_res = Sidekiq.redis do |conn|
67
88
  conn.pipelined do
68
- pipe1_res[7].each {|key| conn.hget(key, 'busy'.freeze) }
69
- pipe1_res[8].each {|queue| conn.llen("queue:#{queue}") }
89
+ processes.each {|key| conn.hget(key, 'busy') }
90
+ queues.each {|queue| conn.llen("queue:#{queue}") }
70
91
  end
71
92
  end
72
93
 
73
- s = pipe1_res[7].size
94
+ s = processes.size
74
95
  workers_size = pipe2_res[0...s].map(&:to_i).inject(0, &:+)
75
96
  enqueued = pipe2_res[s..-1].map(&:to_i).inject(0, &:+)
76
97
 
77
98
  default_queue_latency = if (entry = pipe1_res[6].first)
78
- Time.now.to_f - Sidekiq.load_json(entry)['enqueued_at'.freeze]
99
+ job = Sidekiq.load_json(entry) rescue {}
100
+ now = Time.now.to_f
101
+ thence = job['enqueued_at'] || now
102
+ now - thence
79
103
  else
80
104
  0
81
105
  end
@@ -114,9 +138,11 @@ module Sidekiq
114
138
  end
115
139
 
116
140
  class Queues
141
+ include RedisScanner
142
+
117
143
  def lengths
118
144
  Sidekiq.redis do |conn|
119
- queues = conn.smembers('queues'.freeze)
145
+ queues = sscan(conn, 'queues')
120
146
 
121
147
  lengths = conn.pipelined do
122
148
  queues.each do |queue|
@@ -143,11 +169,11 @@ module Sidekiq
143
169
  end
144
170
 
145
171
  def processed
146
- date_stat_hash("processed")
172
+ @processed ||= date_stat_hash("processed")
147
173
  end
148
174
 
149
175
  def failed
150
- date_stat_hash("failed")
176
+ @failed ||= date_stat_hash("failed")
151
177
  end
152
178
 
153
179
  private
@@ -160,16 +186,21 @@ module Sidekiq
160
186
 
161
187
  while i < @days_previous
162
188
  date = @start_date - i
163
- datestr = date.strftime("%Y-%m-%d".freeze)
189
+ datestr = date.strftime("%Y-%m-%d")
164
190
  keys << "stat:#{stat}:#{datestr}"
165
191
  dates << datestr
166
192
  i += 1
167
193
  end
168
194
 
169
- Sidekiq.redis do |conn|
170
- conn.mget(keys).each_with_index do |value, idx|
171
- stat_hash[dates[idx]] = value ? value.to_i : 0
195
+ begin
196
+ Sidekiq.redis do |conn|
197
+ conn.mget(keys).each_with_index do |value, idx|
198
+ stat_hash[dates[idx]] = value ? value.to_i : 0
199
+ end
172
200
  end
201
+ rescue Redis::CommandError
202
+ # mget will trigger a CROSSSLOT error when run against a Cluster
203
+ # TODO Someone want to add Cluster support?
173
204
  end
174
205
 
175
206
  stat_hash
@@ -191,18 +222,19 @@ module Sidekiq
191
222
  #
192
223
  class Queue
193
224
  include Enumerable
225
+ extend RedisScanner
194
226
 
195
227
  ##
196
228
  # Return all known queues within Redis.
197
229
  #
198
230
  def self.all
199
- Sidekiq.redis { |c| c.smembers('queues'.freeze) }.sort.map { |q| Sidekiq::Queue.new(q) }
231
+ Sidekiq.redis { |c| sscan(c, 'queues') }.sort.map { |q| Sidekiq::Queue.new(q) }
200
232
  end
201
233
 
202
234
  attr_reader :name
203
235
 
204
236
  def initialize(name="default")
205
- @name = name
237
+ @name = name.to_s
206
238
  @rname = "queue:#{name}"
207
239
  end
208
240
 
@@ -225,7 +257,10 @@ module Sidekiq
225
257
  conn.lrange(@rname, -1, -1)
226
258
  end.first
227
259
  return 0 unless entry
228
- Time.now.to_f - Sidekiq.load_json(entry)['enqueued_at']
260
+ job = Sidekiq.load_json(entry)
261
+ now = Time.now.to_f
262
+ thence = job['enqueued_at'] || now
263
+ now - thence
229
264
  end
230
265
 
231
266
  def each
@@ -262,7 +297,7 @@ module Sidekiq
262
297
  Sidekiq.redis do |conn|
263
298
  conn.multi do
264
299
  conn.del(@rname)
265
- conn.srem("queues".freeze, name)
300
+ conn.srem("queues", name)
266
301
  end
267
302
  end
268
303
  end
@@ -281,13 +316,25 @@ module Sidekiq
281
316
  attr_reader :value
282
317
 
283
318
  def initialize(item, queue_name=nil)
319
+ @args = nil
284
320
  @value = item
285
- @item = item.is_a?(Hash) ? item : Sidekiq.load_json(item)
321
+ @item = item.is_a?(Hash) ? item : parse(item)
286
322
  @queue = queue_name || @item['queue']
287
323
  end
288
324
 
325
+ def parse(item)
326
+ Sidekiq.load_json(item)
327
+ rescue JSON::ParserError
328
+ # If the job payload in Redis is invalid JSON, we'll load
329
+ # the item as an empty hash and store the invalid JSON as
330
+ # the job 'args' for display in the Web UI.
331
+ @invalid = true
332
+ @args = [item]
333
+ {}
334
+ end
335
+
289
336
  def klass
290
- @item['class']
337
+ self['class']
291
338
  end
292
339
 
293
340
  def display_class
@@ -312,38 +359,42 @@ module Sidekiq
312
359
 
313
360
  def display_args
314
361
  # Unwrap known wrappers so they show up in a human-friendly manner in the Web UI
315
- @args ||= case klass
362
+ @display_args ||= case klass
316
363
  when /\ASidekiq::Extensions::Delayed/
317
364
  safe_load(args[0], args) do |_, _, arg|
318
365
  arg
319
366
  end
320
367
  when "ActiveJob::QueueAdapters::SidekiqAdapter::JobWrapper"
321
- job_args = @item['wrapped'] ? args[0]["arguments"] : []
322
- if 'ActionMailer::DeliveryJob' == (@item['wrapped'] || args[0])
323
- # remove MailerClass, mailer_method and 'deliver_now'
324
- job_args.drop(3)
368
+ job_args = self['wrapped'] ? args[0]["arguments"] : []
369
+ if 'ActionMailer::DeliveryJob' == (self['wrapped'] || args[0])
370
+ # remove MailerClass, mailer_method and 'deliver_now'
371
+ job_args.drop(3)
325
372
  else
326
- job_args
373
+ job_args
327
374
  end
328
375
  else
376
+ if self['encrypt']
377
+ # no point in showing 150+ bytes of random garbage
378
+ args[-1] = '[encrypted data]'
379
+ end
329
380
  args
330
381
  end
331
382
  end
332
383
 
333
384
  def args
334
- @item['args']
385
+ @args || @item['args']
335
386
  end
336
387
 
337
388
  def jid
338
- @item['jid']
389
+ self['jid']
339
390
  end
340
391
 
341
392
  def enqueued_at
342
- @item['enqueued_at'] ? Time.at(@item['enqueued_at']).utc : nil
393
+ self['enqueued_at'] ? Time.at(self['enqueued_at']).utc : nil
343
394
  end
344
395
 
345
396
  def created_at
346
- Time.at(@item['created_at'] || @item['enqueued_at'] || 0).utc
397
+ Time.at(self['created_at'] || self['enqueued_at'] || 0).utc
347
398
  end
348
399
 
349
400
  def queue
@@ -351,7 +402,8 @@ module Sidekiq
351
402
  end
352
403
 
353
404
  def latency
354
- Time.now.to_f - (@item['enqueued_at'] || @item['created_at'])
405
+ now = Time.now.to_f
406
+ now - (@item['enqueued_at'] || @item['created_at'] || now)
355
407
  end
356
408
 
357
409
  ##
@@ -364,7 +416,10 @@ module Sidekiq
364
416
  end
365
417
 
366
418
  def [](name)
367
- @item[name]
419
+ # nil will happen if the JSON fails to parse.
420
+ # We don't guarantee Sidekiq will work with bad job JSON but we should
421
+ # make a best effort to minimize the damage.
422
+ @item ? @item[name] : nil
368
423
  end
369
424
 
370
425
  private
@@ -416,10 +471,9 @@ module Sidekiq
416
471
  end
417
472
 
418
473
  def retry
419
- raise "Retry not available on jobs which have not failed" unless item["failed_at"]
420
474
  remove_job do |message|
421
475
  msg = Sidekiq.load_json(message)
422
- msg['retry_count'] -= 1
476
+ msg['retry_count'] -= 1 if msg['retry_count']
423
477
  Sidekiq::Client.push(msg)
424
478
  end
425
479
  end
@@ -427,20 +481,15 @@ module Sidekiq
427
481
  ##
428
482
  # Place job in the dead set
429
483
  def kill
430
- raise 'Kill not available on jobs which have not failed' unless item['failed_at']
431
484
  remove_job do |message|
432
- Sidekiq.logger.info { "Killing job #{message['jid']}" }
433
- now = Time.now.to_f
434
- Sidekiq.redis do |conn|
435
- conn.multi do
436
- conn.zadd('dead', now, message)
437
- conn.zremrangebyscore('dead', '-inf', now - DeadSet.timeout)
438
- conn.zremrangebyrank('dead', 0, - DeadSet.max_jobs)
439
- end
440
- end
485
+ DeadSet.new.kill(message)
441
486
  end
442
487
  end
443
488
 
489
+ def error?
490
+ !!item['error_class']
491
+ end
492
+
444
493
  private
445
494
 
446
495
  def remove_job
@@ -523,7 +572,7 @@ module Sidekiq
523
572
  end
524
573
  break if elements.empty?
525
574
  page -= 1
526
- elements.each do |element, score|
575
+ elements.reverse.each do |element, score|
527
576
  yield SortedEntry.new(self, score, element)
528
577
  end
529
578
  offset_size = initial_size - @_size
@@ -585,13 +634,13 @@ module Sidekiq
585
634
  # Allows enumeration of scheduled jobs within Sidekiq.
586
635
  # Based on this, you can search/filter for jobs. Here's an
587
636
  # example where I'm selecting all jobs of a certain type
588
- # and deleting them from the retry queue.
637
+ # and deleting them from the schedule queue.
589
638
  #
590
639
  # r = Sidekiq::ScheduledSet.new
591
- # r.select do |retri|
592
- # retri.klass == 'Sidekiq::Extensions::DelayedClass' &&
593
- # retri.args[0] == 'User' &&
594
- # retri.args[1] == 'setup_new_subscriber'
640
+ # r.select do |scheduled|
641
+ # scheduled.klass == 'Sidekiq::Extensions::DelayedClass' &&
642
+ # scheduled.args[0] == 'User' &&
643
+ # scheduled.args[1] == 'setup_new_subscriber'
595
644
  # end.map(&:delete)
596
645
  class ScheduledSet < JobSet
597
646
  def initialize
@@ -621,6 +670,12 @@ module Sidekiq
621
670
  each(&:retry)
622
671
  end
623
672
  end
673
+
674
+ def kill_all
675
+ while size > 0
676
+ each(&:kill)
677
+ end
678
+ end
624
679
  end
625
680
 
626
681
  ##
@@ -631,6 +686,27 @@ module Sidekiq
631
686
  super 'dead'
632
687
  end
633
688
 
689
+ def kill(message, opts={})
690
+ now = Time.now.to_f
691
+ Sidekiq.redis do |conn|
692
+ conn.multi do
693
+ conn.zadd(name, now.to_s, message)
694
+ conn.zremrangebyscore(name, '-inf', now - self.class.timeout)
695
+ conn.zremrangebyrank(name, 0, - self.class.max_jobs)
696
+ end
697
+ end
698
+
699
+ if opts[:notify_failure] != false
700
+ job = Sidekiq.load_json(message)
701
+ r = RuntimeError.new("Job killed by API")
702
+ r.set_backtrace(caller)
703
+ Sidekiq.death_handlers.each do |handle|
704
+ handle.call(job, r)
705
+ end
706
+ end
707
+ true
708
+ end
709
+
634
710
  def retry_all
635
711
  while size > 0
636
712
  each(&:retry)
@@ -655,17 +731,18 @@ module Sidekiq
655
731
  #
656
732
  class ProcessSet
657
733
  include Enumerable
734
+ include RedisScanner
658
735
 
659
736
  def initialize(clean_plz=true)
660
- self.class.cleanup if clean_plz
737
+ cleanup if clean_plz
661
738
  end
662
739
 
663
740
  # Cleans up dead processes recorded in Redis.
664
741
  # Returns the number of processes cleaned.
665
- def self.cleanup
742
+ def cleanup
666
743
  count = 0
667
744
  Sidekiq.redis do |conn|
668
- procs = conn.smembers('processes').sort
745
+ procs = sscan(conn, 'processes').sort
669
746
  heartbeats = conn.pipelined do
670
747
  procs.each do |key|
671
748
  conn.hget(key, 'info')
@@ -685,7 +762,7 @@ module Sidekiq
685
762
  end
686
763
 
687
764
  def each
688
- procs = Sidekiq.redis { |conn| conn.smembers('processes') }.sort
765
+ procs = Sidekiq.redis { |conn| sscan(conn, 'processes') }.sort
689
766
 
690
767
  Sidekiq.redis do |conn|
691
768
  # We're making a tradeoff here between consuming more memory instead of
@@ -698,6 +775,11 @@ module Sidekiq
698
775
  end
699
776
 
700
777
  result.each do |info, busy, at_s, quiet|
778
+ # If a process is stopped between when we query Redis for `procs` and
779
+ # when we query for `result`, we will have an item in `result` that is
780
+ # composed of `nil` values.
781
+ next if info.nil?
782
+
701
783
  hash = Sidekiq.load_json(info)
702
784
  yield Process.new(hash.merge('busy' => busy.to_i, 'beat' => at_s.to_f, 'quiet' => quiet))
703
785
  end
@@ -713,6 +795,18 @@ module Sidekiq
713
795
  def size
714
796
  Sidekiq.redis { |conn| conn.scard('processes') }
715
797
  end
798
+
799
+ # Returns the identity of the current cluster leader or "" if no leader.
800
+ # This is a Sidekiq Enterprise feature, will always return "" in Sidekiq
801
+ # or Sidekiq Pro.
802
+ def leader
803
+ @leader ||= begin
804
+ x = Sidekiq.redis {|c| c.get("dear-leader") }
805
+ # need a non-falsy value so we can memoize
806
+ x = "" unless x
807
+ x
808
+ end
809
+ end
716
810
  end
717
811
 
718
812
  #
@@ -747,8 +841,12 @@ module Sidekiq
747
841
  @attribs[key]
748
842
  end
749
843
 
844
+ def identity
845
+ self['identity']
846
+ end
847
+
750
848
  def quiet!
751
- signal('USR1')
849
+ signal('TSTP')
752
850
  end
753
851
 
754
852
  def stop!
@@ -775,9 +873,6 @@ module Sidekiq
775
873
  end
776
874
  end
777
875
 
778
- def identity
779
- self['identity']
780
- end
781
876
  end
782
877
 
783
878
  ##
@@ -802,10 +897,11 @@ module Sidekiq
802
897
  #
803
898
  class Workers
804
899
  include Enumerable
900
+ include RedisScanner
805
901
 
806
902
  def each
807
903
  Sidekiq.redis do |conn|
808
- procs = conn.smembers('processes')
904
+ procs = sscan(conn, 'processes')
809
905
  procs.sort.each do |key|
810
906
  valid, workers = conn.pipelined do
811
907
  conn.exists(key)
@@ -827,7 +923,7 @@ module Sidekiq
827
923
  # which can easily get out of sync with crashy processes.
828
924
  def size
829
925
  Sidekiq.redis do |conn|
830
- procs = conn.smembers('processes')
926
+ procs = sscan(conn, 'processes')
831
927
  if procs.empty?
832
928
  0
833
929
  else