sidekiq 6.0.0 → 6.0.3
Sign up to get free protection for your applications and to get access to all the features.
Potentially problematic release.
This version of sidekiq might be problematic. Click here for more details.
- checksums.yaml +4 -4
- data/.circleci/config.yml +21 -0
- data/6.0-Upgrade.md +3 -1
- data/Changes.md +88 -1
- data/Ent-Changes.md +6 -0
- data/Gemfile.lock +3 -3
- data/Pro-Changes.md +9 -1
- data/README.md +3 -1
- data/bin/sidekiqload +8 -4
- data/bin/sidekiqmon +4 -5
- data/lib/generators/sidekiq/worker_generator.rb +10 -0
- data/lib/sidekiq/api.rb +117 -88
- data/lib/sidekiq/cli.rb +19 -17
- data/lib/sidekiq/client.rb +12 -2
- data/lib/sidekiq/fetch.rb +7 -7
- data/lib/sidekiq/job_logger.rb +11 -3
- data/lib/sidekiq/job_retry.rb +21 -8
- data/lib/sidekiq/launcher.rb +1 -3
- data/lib/sidekiq/logger.rb +107 -11
- data/lib/sidekiq/middleware/chain.rb +11 -2
- data/lib/sidekiq/monitor.rb +1 -16
- data/lib/sidekiq/paginator.rb +7 -2
- data/lib/sidekiq/processor.rb +17 -19
- data/lib/sidekiq/scheduled.rb +13 -12
- data/lib/sidekiq/testing.rb +12 -0
- data/lib/sidekiq/util.rb +0 -2
- data/lib/sidekiq/version.rb +1 -1
- data/lib/sidekiq/web/application.rb +8 -13
- data/lib/sidekiq/web/helpers.rb +22 -10
- data/lib/sidekiq/worker.rb +4 -4
- data/lib/sidekiq.rb +8 -0
- data/sidekiq.gemspec +1 -1
- data/web/assets/javascripts/dashboard.js +2 -2
- data/web/assets/stylesheets/application-dark.css +125 -0
- data/web/assets/stylesheets/application.css +9 -0
- data/web/locales/de.yml +14 -2
- data/web/views/_job_info.erb +2 -1
- data/web/views/busy.erb +4 -1
- data/web/views/dead.erb +2 -2
- data/web/views/layout.erb +1 -0
- data/web/views/morgue.erb +4 -1
- data/web/views/queue.erb +10 -1
- data/web/views/retries.erb +4 -1
- data/web/views/retry.erb +2 -2
- data/web/views/scheduled.erb +4 -1
- metadata +5 -4
data/lib/sidekiq/api.rb
CHANGED
@@ -2,23 +2,11 @@
|
|
2
2
|
|
3
3
|
require "sidekiq"
|
4
4
|
|
5
|
-
|
6
|
-
|
7
|
-
def sscan(conn, key)
|
8
|
-
cursor = "0"
|
9
|
-
result = []
|
10
|
-
loop do
|
11
|
-
cursor, values = conn.sscan(key, cursor)
|
12
|
-
result.push(*values)
|
13
|
-
break if cursor == "0"
|
14
|
-
end
|
15
|
-
result
|
16
|
-
end
|
17
|
-
end
|
5
|
+
require "zlib"
|
6
|
+
require "base64"
|
18
7
|
|
8
|
+
module Sidekiq
|
19
9
|
class Stats
|
20
|
-
include RedisScanner
|
21
|
-
|
22
10
|
def initialize
|
23
11
|
fetch_stats!
|
24
12
|
end
|
@@ -77,11 +65,11 @@ module Sidekiq
|
|
77
65
|
}
|
78
66
|
|
79
67
|
processes = Sidekiq.redis { |conn|
|
80
|
-
|
68
|
+
conn.sscan_each("processes").to_a
|
81
69
|
}
|
82
70
|
|
83
71
|
queues = Sidekiq.redis { |conn|
|
84
|
-
|
72
|
+
conn.sscan_each("queues").to_a
|
85
73
|
}
|
86
74
|
|
87
75
|
pipe2_res = Sidekiq.redis { |conn|
|
@@ -92,8 +80,8 @@ module Sidekiq
|
|
92
80
|
}
|
93
81
|
|
94
82
|
s = processes.size
|
95
|
-
workers_size = pipe2_res[0...s].
|
96
|
-
enqueued = pipe2_res[s..-1].
|
83
|
+
workers_size = pipe2_res[0...s].sum(&:to_i)
|
84
|
+
enqueued = pipe2_res[s..-1].sum(&:to_i)
|
97
85
|
|
98
86
|
default_queue_latency = if (entry = pipe1_res[6].first)
|
99
87
|
job = begin
|
@@ -142,11 +130,9 @@ module Sidekiq
|
|
142
130
|
end
|
143
131
|
|
144
132
|
class Queues
|
145
|
-
include RedisScanner
|
146
|
-
|
147
133
|
def lengths
|
148
134
|
Sidekiq.redis do |conn|
|
149
|
-
queues =
|
135
|
+
queues = conn.sscan_each("queues").to_a
|
150
136
|
|
151
137
|
lengths = conn.pipelined {
|
152
138
|
queues.each do |queue|
|
@@ -154,13 +140,8 @@ module Sidekiq
|
|
154
140
|
end
|
155
141
|
}
|
156
142
|
|
157
|
-
|
158
|
-
array_of_arrays
|
159
|
-
memo[queue] = lengths[i]
|
160
|
-
i += 1
|
161
|
-
}.sort_by { |_, size| size }
|
162
|
-
|
163
|
-
Hash[array_of_arrays.reverse]
|
143
|
+
array_of_arrays = queues.zip(lengths).sort_by { |_, size| -size }
|
144
|
+
Hash[array_of_arrays]
|
164
145
|
end
|
165
146
|
end
|
166
147
|
end
|
@@ -182,18 +163,12 @@ module Sidekiq
|
|
182
163
|
private
|
183
164
|
|
184
165
|
def date_stat_hash(stat)
|
185
|
-
i = 0
|
186
166
|
stat_hash = {}
|
187
|
-
|
188
|
-
|
189
|
-
|
190
|
-
|
191
|
-
|
192
|
-
datestr = date.strftime("%Y-%m-%d")
|
193
|
-
keys << "stat:#{stat}:#{datestr}"
|
194
|
-
dates << datestr
|
195
|
-
i += 1
|
196
|
-
end
|
167
|
+
dates = @start_date.downto(@start_date - @days_previous + 1).map { |date|
|
168
|
+
date.strftime("%Y-%m-%d")
|
169
|
+
}
|
170
|
+
|
171
|
+
keys = dates.map { |datestr| "stat:#{stat}:#{datestr}" }
|
197
172
|
|
198
173
|
begin
|
199
174
|
Sidekiq.redis do |conn|
|
@@ -225,13 +200,12 @@ module Sidekiq
|
|
225
200
|
#
|
226
201
|
class Queue
|
227
202
|
include Enumerable
|
228
|
-
extend RedisScanner
|
229
203
|
|
230
204
|
##
|
231
205
|
# Return all known queues within Redis.
|
232
206
|
#
|
233
207
|
def self.all
|
234
|
-
Sidekiq.redis { |c|
|
208
|
+
Sidekiq.redis { |c| c.sscan_each("queues").to_a }.sort.map { |q| Sidekiq::Queue.new(q) }
|
235
209
|
end
|
236
210
|
|
237
211
|
attr_reader :name
|
@@ -349,7 +323,7 @@ module Sidekiq
|
|
349
323
|
end
|
350
324
|
when "ActiveJob::QueueAdapters::SidekiqAdapter::JobWrapper"
|
351
325
|
job_class = @item["wrapped"] || args[0]
|
352
|
-
if job_class == "ActionMailer::DeliveryJob"
|
326
|
+
if job_class == "ActionMailer::DeliveryJob" || job_class == "ActionMailer::MailDeliveryJob"
|
353
327
|
# MailerClass#mailer_method
|
354
328
|
args[0]["arguments"][0..1].join("#")
|
355
329
|
else
|
@@ -372,6 +346,9 @@ module Sidekiq
|
|
372
346
|
if (self["wrapped"] || args[0]) == "ActionMailer::DeliveryJob"
|
373
347
|
# remove MailerClass, mailer_method and 'deliver_now'
|
374
348
|
job_args.drop(3)
|
349
|
+
elsif (self["wrapped"] || args[0]) == "ActionMailer::MailDeliveryJob"
|
350
|
+
# remove MailerClass, mailer_method and 'deliver_now'
|
351
|
+
job_args.drop(3).first["args"]
|
375
352
|
else
|
376
353
|
job_args
|
377
354
|
end
|
@@ -400,6 +377,20 @@ module Sidekiq
|
|
400
377
|
Time.at(self["created_at"] || self["enqueued_at"] || 0).utc
|
401
378
|
end
|
402
379
|
|
380
|
+
def tags
|
381
|
+
self["tags"] || []
|
382
|
+
end
|
383
|
+
|
384
|
+
def error_backtrace
|
385
|
+
# Cache nil values
|
386
|
+
if defined?(@error_backtrace)
|
387
|
+
@error_backtrace
|
388
|
+
else
|
389
|
+
value = self["error_backtrace"]
|
390
|
+
@error_backtrace = value && uncompress_backtrace(value)
|
391
|
+
end
|
392
|
+
end
|
393
|
+
|
403
394
|
attr_reader :queue
|
404
395
|
|
405
396
|
def latency
|
@@ -433,6 +424,23 @@ module Sidekiq
|
|
433
424
|
Sidekiq.logger.warn "Unable to load YAML: #{ex.message}" unless Sidekiq.options[:environment] == "development"
|
434
425
|
default
|
435
426
|
end
|
427
|
+
|
428
|
+
def uncompress_backtrace(backtrace)
|
429
|
+
if backtrace.is_a?(Array)
|
430
|
+
# Handle old jobs with raw Array backtrace format
|
431
|
+
backtrace
|
432
|
+
else
|
433
|
+
decoded = Base64.decode64(backtrace)
|
434
|
+
uncompressed = Zlib::Inflate.inflate(decoded)
|
435
|
+
begin
|
436
|
+
Sidekiq.load_json(uncompressed)
|
437
|
+
rescue
|
438
|
+
# Handle old jobs with marshalled backtrace format
|
439
|
+
# TODO Remove in 7.x
|
440
|
+
Marshal.load(uncompressed)
|
441
|
+
end
|
442
|
+
end
|
443
|
+
end
|
436
444
|
end
|
437
445
|
|
438
446
|
class SortedEntry < Job
|
@@ -458,8 +466,9 @@ module Sidekiq
|
|
458
466
|
end
|
459
467
|
|
460
468
|
def reschedule(at)
|
461
|
-
|
462
|
-
|
469
|
+
Sidekiq.redis do |conn|
|
470
|
+
conn.zincrby(@parent.name, at.to_f - @score, Sidekiq.dump_json(@item))
|
471
|
+
end
|
463
472
|
end
|
464
473
|
|
465
474
|
def add_to_queue
|
@@ -503,7 +512,7 @@ module Sidekiq
|
|
503
512
|
else
|
504
513
|
# multiple jobs with the same score
|
505
514
|
# find the one with the right JID and push it
|
506
|
-
|
515
|
+
matched, nonmatched = results.partition { |message|
|
507
516
|
if message.index(jid)
|
508
517
|
msg = Sidekiq.load_json(message)
|
509
518
|
msg["jid"] == jid
|
@@ -512,12 +521,12 @@ module Sidekiq
|
|
512
521
|
end
|
513
522
|
}
|
514
523
|
|
515
|
-
msg =
|
524
|
+
msg = matched.first
|
516
525
|
yield msg if msg
|
517
526
|
|
518
527
|
# push the rest back onto the sorted set
|
519
528
|
conn.multi do
|
520
|
-
|
529
|
+
nonmatched.each do |message|
|
521
530
|
conn.zadd(parent.name, score.to_f.to_s, message)
|
522
531
|
end
|
523
532
|
end
|
@@ -540,6 +549,17 @@ module Sidekiq
|
|
540
549
|
Sidekiq.redis { |c| c.zcard(name) }
|
541
550
|
end
|
542
551
|
|
552
|
+
def scan(match, count = 100)
|
553
|
+
return to_enum(:scan, match, count) unless block_given?
|
554
|
+
|
555
|
+
match = "*#{match}*" unless match.include?("*")
|
556
|
+
Sidekiq.redis do |conn|
|
557
|
+
conn.zscan_each(name, match: match, count: count) do |entry, score|
|
558
|
+
yield SortedEntry.new(self, score, entry)
|
559
|
+
end
|
560
|
+
end
|
561
|
+
end
|
562
|
+
|
543
563
|
def clear
|
544
564
|
Sidekiq.redis do |conn|
|
545
565
|
conn.del(name)
|
@@ -576,28 +596,40 @@ module Sidekiq
|
|
576
596
|
end
|
577
597
|
end
|
578
598
|
|
599
|
+
##
|
600
|
+
# Fetch jobs that match a given time or Range. Job ID is an
|
601
|
+
# optional second argument.
|
579
602
|
def fetch(score, jid = nil)
|
603
|
+
begin_score, end_score =
|
604
|
+
if score.is_a?(Range)
|
605
|
+
[score.first, score.last]
|
606
|
+
else
|
607
|
+
[score, score]
|
608
|
+
end
|
609
|
+
|
580
610
|
elements = Sidekiq.redis { |conn|
|
581
|
-
conn.zrangebyscore(name,
|
611
|
+
conn.zrangebyscore(name, begin_score, end_score, with_scores: true)
|
582
612
|
}
|
583
613
|
|
584
614
|
elements.each_with_object([]) do |element, result|
|
585
|
-
|
586
|
-
|
587
|
-
|
588
|
-
else
|
589
|
-
result << entry
|
590
|
-
end
|
615
|
+
data, job_score = element
|
616
|
+
entry = SortedEntry.new(self, job_score, data)
|
617
|
+
result << entry if jid.nil? || entry.jid == jid
|
591
618
|
end
|
592
619
|
end
|
593
620
|
|
594
621
|
##
|
595
622
|
# Find the job with the given JID within this sorted set.
|
596
|
-
#
|
597
|
-
# This is a slow, inefficient operation. Do not use under
|
598
|
-
# normal conditions. Sidekiq Pro contains a faster version.
|
623
|
+
# This is a slower O(n) operation. Do not use for app logic.
|
599
624
|
def find_job(jid)
|
600
|
-
|
625
|
+
Sidekiq.redis do |conn|
|
626
|
+
conn.zscan_each(name, match: "*#{jid}*", count: 100) do |entry, score|
|
627
|
+
job = JSON.parse(entry)
|
628
|
+
matched = job["jid"] == jid
|
629
|
+
return SortedEntry.new(self, score, entry) if matched
|
630
|
+
end
|
631
|
+
end
|
632
|
+
nil
|
601
633
|
end
|
602
634
|
|
603
635
|
def delete_by_value(name, value)
|
@@ -612,11 +644,13 @@ module Sidekiq
|
|
612
644
|
Sidekiq.redis do |conn|
|
613
645
|
elements = conn.zrangebyscore(name, score, score)
|
614
646
|
elements.each do |element|
|
615
|
-
|
616
|
-
|
617
|
-
|
618
|
-
|
619
|
-
|
647
|
+
if element.index(jid)
|
648
|
+
message = Sidekiq.load_json(element)
|
649
|
+
if message["jid"] == jid
|
650
|
+
ret = conn.zrem(name, element)
|
651
|
+
@_size -= 1 if ret
|
652
|
+
break ret
|
653
|
+
end
|
620
654
|
end
|
621
655
|
end
|
622
656
|
end
|
@@ -720,7 +754,6 @@ module Sidekiq
|
|
720
754
|
#
|
721
755
|
class ProcessSet
|
722
756
|
include Enumerable
|
723
|
-
include RedisScanner
|
724
757
|
|
725
758
|
def initialize(clean_plz = true)
|
726
759
|
cleanup if clean_plz
|
@@ -731,7 +764,7 @@ module Sidekiq
|
|
731
764
|
def cleanup
|
732
765
|
count = 0
|
733
766
|
Sidekiq.redis do |conn|
|
734
|
-
procs =
|
767
|
+
procs = conn.sscan_each("processes").to_a.sort
|
735
768
|
heartbeats = conn.pipelined {
|
736
769
|
procs.each do |key|
|
737
770
|
conn.hget(key, "info")
|
@@ -741,40 +774,37 @@ module Sidekiq
|
|
741
774
|
# the hash named key has an expiry of 60 seconds.
|
742
775
|
# if it's not found, that means the process has not reported
|
743
776
|
# in to Redis and probably died.
|
744
|
-
to_prune =
|
745
|
-
|
746
|
-
|
747
|
-
end
|
777
|
+
to_prune = procs.select.with_index { |proc, i|
|
778
|
+
heartbeats[i].nil?
|
779
|
+
}
|
748
780
|
count = conn.srem("processes", to_prune) unless to_prune.empty?
|
749
781
|
end
|
750
782
|
count
|
751
783
|
end
|
752
784
|
|
753
785
|
def each
|
754
|
-
|
786
|
+
result = Sidekiq.redis { |conn|
|
787
|
+
procs = conn.sscan_each("processes").to_a.sort
|
755
788
|
|
756
|
-
Sidekiq.redis do |conn|
|
757
789
|
# We're making a tradeoff here between consuming more memory instead of
|
758
790
|
# making more roundtrips to Redis, but if you have hundreds or thousands of workers,
|
759
791
|
# you'll be happier this way
|
760
|
-
|
792
|
+
conn.pipelined do
|
761
793
|
procs.each do |key|
|
762
794
|
conn.hmget(key, "info", "busy", "beat", "quiet")
|
763
795
|
end
|
764
|
-
|
796
|
+
end
|
797
|
+
}
|
765
798
|
|
766
|
-
|
767
|
-
|
768
|
-
|
769
|
-
|
770
|
-
|
799
|
+
result.each do |info, busy, at_s, quiet|
|
800
|
+
# If a process is stopped between when we query Redis for `procs` and
|
801
|
+
# when we query for `result`, we will have an item in `result` that is
|
802
|
+
# composed of `nil` values.
|
803
|
+
next if info.nil?
|
771
804
|
|
772
|
-
|
773
|
-
|
774
|
-
end
|
805
|
+
hash = Sidekiq.load_json(info)
|
806
|
+
yield Process.new(hash.merge("busy" => busy.to_i, "beat" => at_s.to_f, "quiet" => quiet))
|
775
807
|
end
|
776
|
-
|
777
|
-
nil
|
778
808
|
end
|
779
809
|
|
780
810
|
# This method is not guaranteed accurate since it does not prune the set
|
@@ -885,11 +915,10 @@ module Sidekiq
|
|
885
915
|
#
|
886
916
|
class Workers
|
887
917
|
include Enumerable
|
888
|
-
include RedisScanner
|
889
918
|
|
890
919
|
def each
|
891
920
|
Sidekiq.redis do |conn|
|
892
|
-
procs =
|
921
|
+
procs = conn.sscan_each("processes").to_a
|
893
922
|
procs.sort.each do |key|
|
894
923
|
valid, workers = conn.pipelined {
|
895
924
|
conn.exists(key)
|
@@ -911,7 +940,7 @@ module Sidekiq
|
|
911
940
|
# which can easily get out of sync with crashy processes.
|
912
941
|
def size
|
913
942
|
Sidekiq.redis do |conn|
|
914
|
-
procs =
|
943
|
+
procs = conn.sscan_each("processes").to_a
|
915
944
|
if procs.empty?
|
916
945
|
0
|
917
946
|
else
|
@@ -919,7 +948,7 @@ module Sidekiq
|
|
919
948
|
procs.each do |key|
|
920
949
|
conn.hget(key, "busy")
|
921
950
|
end
|
922
|
-
}.
|
951
|
+
}.sum(&:to_i)
|
923
952
|
end
|
924
953
|
end
|
925
954
|
end
|
data/lib/sidekiq/cli.rb
CHANGED
@@ -41,9 +41,11 @@ module Sidekiq
|
|
41
41
|
|
42
42
|
self_read, self_write = IO.pipe
|
43
43
|
sigs = %w[INT TERM TTIN TSTP]
|
44
|
+
# USR1 and USR2 don't work on the JVM
|
45
|
+
sigs << "USR2" unless jruby?
|
44
46
|
sigs.each do |sig|
|
45
47
|
trap sig do
|
46
|
-
self_write.
|
48
|
+
self_write.puts(sig)
|
47
49
|
end
|
48
50
|
rescue ArgumentError
|
49
51
|
puts "Signal #{sig} not supported"
|
@@ -56,7 +58,7 @@ module Sidekiq
|
|
56
58
|
# touch the connection pool so it is created before we
|
57
59
|
# fire startup and start multithreading.
|
58
60
|
ver = Sidekiq.redis_info["redis_version"]
|
59
|
-
raise "You are
|
61
|
+
raise "You are connecting to Redis v#{ver}, Sidekiq requires Redis v4.0.0 or greater" if ver < "4"
|
60
62
|
|
61
63
|
# Since the user can pass us a connection pool explicitly in the initializer, we
|
62
64
|
# need to verify the size is large enough or else Sidekiq's performance is dramatically slowed.
|
@@ -162,15 +164,12 @@ module Sidekiq
|
|
162
164
|
end
|
163
165
|
},
|
164
166
|
}
|
167
|
+
UNHANDLED_SIGNAL_HANDLER = ->(cli) { Sidekiq.logger.info "No signal handler registered, ignoring" }
|
168
|
+
SIGNAL_HANDLERS.default = UNHANDLED_SIGNAL_HANDLER
|
165
169
|
|
166
170
|
def handle_signal(sig)
|
167
171
|
Sidekiq.logger.debug "Got #{sig} signal"
|
168
|
-
|
169
|
-
if handy
|
170
|
-
handy.call(self)
|
171
|
-
else
|
172
|
-
Sidekiq.logger.info { "No signal handler for #{sig}" }
|
173
|
-
end
|
172
|
+
SIGNAL_HANDLERS[sig].call(self)
|
174
173
|
end
|
175
174
|
|
176
175
|
private
|
@@ -204,7 +203,7 @@ module Sidekiq
|
|
204
203
|
|
205
204
|
# check config file presence
|
206
205
|
if opts[:config_file]
|
207
|
-
|
206
|
+
unless File.exist?(opts[:config_file])
|
208
207
|
raise ArgumentError, "No such file #{opts[:config_file]}"
|
209
208
|
end
|
210
209
|
else
|
@@ -224,7 +223,7 @@ module Sidekiq
|
|
224
223
|
opts = parse_config(opts[:config_file]).merge(opts) if opts[:config_file]
|
225
224
|
|
226
225
|
# set defaults
|
227
|
-
opts[:queues] =
|
226
|
+
opts[:queues] = ["default"] if opts[:queues].nil? || opts[:queues].empty?
|
228
227
|
opts[:strict] = true if opts[:strict].nil?
|
229
228
|
opts[:concurrency] = Integer(ENV["RAILS_MAX_THREADS"]) if opts[:concurrency].nil? && ENV["RAILS_MAX_THREADS"]
|
230
229
|
|
@@ -283,8 +282,13 @@ module Sidekiq
|
|
283
282
|
|
284
283
|
def parse_options(argv)
|
285
284
|
opts = {}
|
285
|
+
@parser = option_parser(opts)
|
286
|
+
@parser.parse!(argv)
|
287
|
+
opts
|
288
|
+
end
|
286
289
|
|
287
|
-
|
290
|
+
def option_parser(opts)
|
291
|
+
parser = OptionParser.new { |o|
|
288
292
|
o.on "-c", "--concurrency INT", "processor threads to use" do |arg|
|
289
293
|
opts[:concurrency] = Integer(arg)
|
290
294
|
end
|
@@ -336,15 +340,13 @@ module Sidekiq
|
|
336
340
|
end
|
337
341
|
}
|
338
342
|
|
339
|
-
|
340
|
-
|
341
|
-
logger.info
|
343
|
+
parser.banner = "sidekiq [options]"
|
344
|
+
parser.on_tail "-h", "--help", "Show help" do
|
345
|
+
logger.info parser
|
342
346
|
die 1
|
343
347
|
end
|
344
348
|
|
345
|
-
|
346
|
-
|
347
|
-
opts
|
349
|
+
parser
|
348
350
|
end
|
349
351
|
|
350
352
|
def initialize_logger
|
data/lib/sidekiq/client.rb
CHANGED
@@ -94,9 +94,14 @@ module Sidekiq
|
|
94
94
|
return [] unless arg # no jobs to push
|
95
95
|
raise ArgumentError, "Bulk arguments must be an Array of Arrays: [[1], [2]]" unless arg.is_a?(Array)
|
96
96
|
|
97
|
+
at = items.delete("at")
|
98
|
+
raise ArgumentError, "Job 'at' must be a Numeric or an Array of Numeric timestamps" if at && (Array(at).empty? || !Array(at).all?(Numeric))
|
99
|
+
|
97
100
|
normed = normalize_item(items)
|
98
|
-
payloads = items["args"].map { |args|
|
101
|
+
payloads = items["args"].map.with_index { |args, index|
|
99
102
|
copy = normed.merge("args" => args, "jid" => SecureRandom.hex(12), "enqueued_at" => Time.now.to_f)
|
103
|
+
copy["at"] = (at.is_a?(Array) ? at[index] : at) if at
|
104
|
+
|
100
105
|
result = process_single(items["class"], copy)
|
101
106
|
result || nil
|
102
107
|
}.compact
|
@@ -188,7 +193,7 @@ module Sidekiq
|
|
188
193
|
end
|
189
194
|
|
190
195
|
def atomic_push(conn, payloads)
|
191
|
-
if payloads.first
|
196
|
+
if payloads.first.key?("at")
|
192
197
|
conn.zadd("schedule", payloads.map { |hash|
|
193
198
|
at = hash.delete("at").to_s
|
194
199
|
[at, Sidekiq.dump_json(hash)]
|
@@ -214,10 +219,15 @@ module Sidekiq
|
|
214
219
|
end
|
215
220
|
|
216
221
|
def normalize_item(item)
|
222
|
+
# 6.0.0 push_bulk bug, #4321
|
223
|
+
# TODO Remove after a while...
|
224
|
+
item.delete("at") if item.key?("at") && item["at"].nil?
|
225
|
+
|
217
226
|
raise(ArgumentError, "Job must be a Hash with 'class' and 'args' keys: { 'class' => SomeWorker, 'args' => ['bob', 1, :foo => 'bar'] }") unless item.is_a?(Hash) && item.key?("class") && item.key?("args")
|
218
227
|
raise(ArgumentError, "Job args must be an Array") unless item["args"].is_a?(Array)
|
219
228
|
raise(ArgumentError, "Job class must be either a Class or String representation of the class name") unless item["class"].is_a?(Class) || item["class"].is_a?(String)
|
220
229
|
raise(ArgumentError, "Job 'at' must be a Numeric timestamp") if item.key?("at") && !item["at"].is_a?(Numeric)
|
230
|
+
raise(ArgumentError, "Job tags must be an Array") if item["tags"] && !item["tags"].is_a?(Array)
|
221
231
|
# raise(ArgumentError, "Arguments must be native JSON types, see https://github.com/mperham/sidekiq/wiki/Best-Practices") unless JSON.load(JSON.dump(item['args'])) == item['args']
|
222
232
|
|
223
233
|
normalized_hash(item["class"])
|
data/lib/sidekiq/fetch.rb
CHANGED
@@ -14,12 +14,12 @@ module Sidekiq
|
|
14
14
|
end
|
15
15
|
|
16
16
|
def queue_name
|
17
|
-
queue.
|
17
|
+
queue.delete_prefix("queue:")
|
18
18
|
end
|
19
19
|
|
20
20
|
def requeue
|
21
21
|
Sidekiq.redis do |conn|
|
22
|
-
conn.rpush(
|
22
|
+
conn.rpush(queue, job)
|
23
23
|
end
|
24
24
|
end
|
25
25
|
}
|
@@ -28,7 +28,7 @@ module Sidekiq
|
|
28
28
|
@strictly_ordered_queues = !!options[:strict]
|
29
29
|
@queues = options[:queues].map { |q| "queue:#{q}" }
|
30
30
|
if @strictly_ordered_queues
|
31
|
-
@queues
|
31
|
+
@queues.uniq!
|
32
32
|
@queues << TIMEOUT
|
33
33
|
end
|
34
34
|
end
|
@@ -47,7 +47,7 @@ module Sidekiq
|
|
47
47
|
if @strictly_ordered_queues
|
48
48
|
@queues
|
49
49
|
else
|
50
|
-
queues = @queues.shuffle
|
50
|
+
queues = @queues.shuffle!.uniq
|
51
51
|
queues << TIMEOUT
|
52
52
|
queues
|
53
53
|
end
|
@@ -61,14 +61,14 @@ module Sidekiq
|
|
61
61
|
Sidekiq.logger.debug { "Re-queueing terminated jobs" }
|
62
62
|
jobs_to_requeue = {}
|
63
63
|
inprogress.each do |unit_of_work|
|
64
|
-
jobs_to_requeue[unit_of_work.
|
65
|
-
jobs_to_requeue[unit_of_work.
|
64
|
+
jobs_to_requeue[unit_of_work.queue] ||= []
|
65
|
+
jobs_to_requeue[unit_of_work.queue] << unit_of_work.job
|
66
66
|
end
|
67
67
|
|
68
68
|
Sidekiq.redis do |conn|
|
69
69
|
conn.pipelined do
|
70
70
|
jobs_to_requeue.each do |queue, jobs|
|
71
|
-
conn.rpush(
|
71
|
+
conn.rpush(queue, jobs)
|
72
72
|
end
|
73
73
|
end
|
74
74
|
end
|
data/lib/sidekiq/job_logger.rb
CHANGED
@@ -23,8 +23,15 @@ module Sidekiq
|
|
23
23
|
raise
|
24
24
|
end
|
25
25
|
|
26
|
-
def
|
27
|
-
|
26
|
+
def prepare(job_hash, &block)
|
27
|
+
level = job_hash["log_level"]
|
28
|
+
if level
|
29
|
+
@logger.log_at(level) do
|
30
|
+
Sidekiq::Context.with(job_hash_context(job_hash), &block)
|
31
|
+
end
|
32
|
+
else
|
33
|
+
Sidekiq::Context.with(job_hash_context(job_hash), &block)
|
34
|
+
end
|
28
35
|
end
|
29
36
|
|
30
37
|
def job_hash_context(job_hash)
|
@@ -35,11 +42,12 @@ module Sidekiq
|
|
35
42
|
jid: job_hash["jid"],
|
36
43
|
}
|
37
44
|
h[:bid] = job_hash["bid"] if job_hash["bid"]
|
45
|
+
h[:tags] = job_hash["tags"] if job_hash["tags"]
|
38
46
|
h
|
39
47
|
end
|
40
48
|
|
41
49
|
def with_elapsed_time_context(start, &block)
|
42
|
-
|
50
|
+
Sidekiq::Context.with(elapsed_time_context(start), &block)
|
43
51
|
end
|
44
52
|
|
45
53
|
def elapsed_time_context(start)
|