sidekiq 6.4.0 → 6.5.1
Sign up to get free protection for your applications and to get access to all the features.
Potentially problematic release.
This version of sidekiq might be problematic. Click here for more details.
- checksums.yaml +4 -4
- data/Changes.md +54 -1
- data/README.md +6 -1
- data/bin/sidekiq +3 -3
- data/bin/sidekiqload +70 -66
- data/bin/sidekiqmon +1 -1
- data/lib/sidekiq/.DS_Store +0 -0
- data/lib/sidekiq/api.rb +109 -78
- data/lib/sidekiq/cli.rb +47 -38
- data/lib/sidekiq/client.rb +42 -28
- data/lib/sidekiq/component.rb +64 -0
- data/lib/sidekiq/delay.rb +2 -2
- data/lib/sidekiq/extensions/action_mailer.rb +2 -2
- data/lib/sidekiq/extensions/active_record.rb +2 -2
- data/lib/sidekiq/extensions/class_methods.rb +2 -2
- data/lib/sidekiq/extensions/generic_proxy.rb +3 -3
- data/lib/sidekiq/fetch.rb +18 -16
- data/lib/sidekiq/job_logger.rb +15 -27
- data/lib/sidekiq/job_retry.rb +29 -28
- data/lib/sidekiq/job_util.rb +15 -9
- data/lib/sidekiq/launcher.rb +54 -52
- data/lib/sidekiq/logger.rb +8 -18
- data/lib/sidekiq/manager.rb +28 -25
- data/lib/sidekiq/middleware/chain.rb +22 -13
- data/lib/sidekiq/middleware/current_attributes.rb +4 -0
- data/lib/sidekiq/middleware/i18n.rb +6 -4
- data/lib/sidekiq/middleware/modules.rb +21 -0
- data/lib/sidekiq/monitor.rb +1 -1
- data/lib/sidekiq/paginator.rb +8 -8
- data/lib/sidekiq/processor.rb +38 -38
- data/lib/sidekiq/rails.rb +15 -8
- data/lib/sidekiq/redis_client_adapter.rb +154 -0
- data/lib/sidekiq/redis_connection.rb +81 -48
- data/lib/sidekiq/ring_buffer.rb +29 -0
- data/lib/sidekiq/scheduled.rb +11 -10
- data/lib/sidekiq/testing/inline.rb +4 -4
- data/lib/sidekiq/testing.rb +37 -36
- data/lib/sidekiq/transaction_aware_client.rb +45 -0
- data/lib/sidekiq/version.rb +1 -1
- data/lib/sidekiq/web/csrf_protection.rb +2 -2
- data/lib/sidekiq/web/helpers.rb +5 -5
- data/lib/sidekiq/web.rb +3 -3
- data/lib/sidekiq/worker.rb +20 -17
- data/lib/sidekiq.rb +98 -30
- data/web/assets/javascripts/application.js +58 -26
- data/web/assets/stylesheets/application.css +1 -0
- data/web/locales/pt-br.yml +27 -9
- data/web/views/_summary.erb +1 -1
- data/web/views/busy.erb +3 -3
- metadata +9 -5
- data/lib/sidekiq/exception_handler.rb +0 -27
- data/lib/sidekiq/util.rb +0 -108
data/lib/sidekiq/api.rb
CHANGED
@@ -54,14 +54,14 @@ module Sidekiq
|
|
54
54
|
# O(1) redis calls
|
55
55
|
def fetch_stats_fast!
|
56
56
|
pipe1_res = Sidekiq.redis { |conn|
|
57
|
-
conn.pipelined do
|
58
|
-
|
59
|
-
|
60
|
-
|
61
|
-
|
62
|
-
|
63
|
-
|
64
|
-
|
57
|
+
conn.pipelined do |pipeline|
|
58
|
+
pipeline.get("stat:processed")
|
59
|
+
pipeline.get("stat:failed")
|
60
|
+
pipeline.zcard("schedule")
|
61
|
+
pipeline.zcard("retry")
|
62
|
+
pipeline.zcard("dead")
|
63
|
+
pipeline.scard("processes")
|
64
|
+
pipeline.lrange("queue:default", -1, -1)
|
65
65
|
end
|
66
66
|
}
|
67
67
|
|
@@ -101,9 +101,9 @@ module Sidekiq
|
|
101
101
|
}
|
102
102
|
|
103
103
|
pipe2_res = Sidekiq.redis { |conn|
|
104
|
-
conn.pipelined do
|
105
|
-
processes.each { |key|
|
106
|
-
queues.each { |queue|
|
104
|
+
conn.pipelined do |pipeline|
|
105
|
+
processes.each { |key| pipeline.hget(key, "busy") }
|
106
|
+
queues.each { |queue| pipeline.llen("queue:#{queue}") }
|
107
107
|
end
|
108
108
|
}
|
109
109
|
|
@@ -147,9 +147,9 @@ module Sidekiq
|
|
147
147
|
Sidekiq.redis do |conn|
|
148
148
|
queues = conn.sscan_each("queues").to_a
|
149
149
|
|
150
|
-
lengths = conn.pipelined {
|
150
|
+
lengths = conn.pipelined { |pipeline|
|
151
151
|
queues.each do |queue|
|
152
|
-
|
152
|
+
pipeline.llen("queue:#{queue}")
|
153
153
|
end
|
154
154
|
}
|
155
155
|
|
@@ -191,7 +191,7 @@ module Sidekiq
|
|
191
191
|
stat_hash[dates[idx]] = value ? value.to_i : 0
|
192
192
|
end
|
193
193
|
end
|
194
|
-
rescue
|
194
|
+
rescue RedisConnection.adapter::CommandError
|
195
195
|
# mget will trigger a CROSSSLOT error when run against a Cluster
|
196
196
|
# TODO Someone want to add Cluster support?
|
197
197
|
end
|
@@ -217,24 +217,30 @@ module Sidekiq
|
|
217
217
|
include Enumerable
|
218
218
|
|
219
219
|
##
|
220
|
-
#
|
220
|
+
# Fetch all known queues within Redis.
|
221
221
|
#
|
222
|
+
# @return [Array<Sidekiq::Queue>]
|
222
223
|
def self.all
|
223
224
|
Sidekiq.redis { |c| c.sscan_each("queues").to_a }.sort.map { |q| Sidekiq::Queue.new(q) }
|
224
225
|
end
|
225
226
|
|
226
227
|
attr_reader :name
|
227
228
|
|
229
|
+
# @param name [String] the name of the queue
|
228
230
|
def initialize(name = "default")
|
229
231
|
@name = name.to_s
|
230
232
|
@rname = "queue:#{name}"
|
231
233
|
end
|
232
234
|
|
235
|
+
# The current size of the queue within Redis.
|
236
|
+
# This value is real-time and can change between calls.
|
237
|
+
#
|
238
|
+
# @return [Integer] the size
|
233
239
|
def size
|
234
240
|
Sidekiq.redis { |con| con.llen(@rname) }
|
235
241
|
end
|
236
242
|
|
237
|
-
#
|
243
|
+
# @return [Boolean] if the queue is currently paused
|
238
244
|
def paused?
|
239
245
|
false
|
240
246
|
end
|
@@ -243,7 +249,7 @@ module Sidekiq
|
|
243
249
|
# Calculates this queue's latency, the difference in seconds since the oldest
|
244
250
|
# job in the queue was enqueued.
|
245
251
|
#
|
246
|
-
# @return Float
|
252
|
+
# @return [Float] in seconds
|
247
253
|
def latency
|
248
254
|
entry = Sidekiq.redis { |conn|
|
249
255
|
conn.lrange(@rname, -1, -1)
|
@@ -279,21 +285,30 @@ module Sidekiq
|
|
279
285
|
##
|
280
286
|
# Find the job with the given JID within this queue.
|
281
287
|
#
|
282
|
-
# This is a slow, inefficient operation. Do not use under
|
288
|
+
# This is a *slow, inefficient* operation. Do not use under
|
283
289
|
# normal conditions.
|
290
|
+
#
|
291
|
+
# @param jid [String] the job_id to look for
|
292
|
+
# @return [Sidekiq::JobRecord]
|
293
|
+
# @return [nil] if not found
|
284
294
|
def find_job(jid)
|
285
295
|
detect { |j| j.jid == jid }
|
286
296
|
end
|
287
297
|
|
298
|
+
# delete all jobs within this queue
|
288
299
|
def clear
|
289
300
|
Sidekiq.redis do |conn|
|
290
|
-
conn.multi do
|
291
|
-
|
292
|
-
|
301
|
+
conn.multi do |transaction|
|
302
|
+
transaction.unlink(@rname)
|
303
|
+
transaction.srem("queues", name)
|
293
304
|
end
|
294
305
|
end
|
295
306
|
end
|
296
307
|
alias_method :💣, :clear
|
308
|
+
|
309
|
+
def as_json(options = nil) # :nodoc:
|
310
|
+
{name: name} # 5336
|
311
|
+
end
|
297
312
|
end
|
298
313
|
|
299
314
|
##
|
@@ -306,15 +321,16 @@ module Sidekiq
|
|
306
321
|
class JobRecord
|
307
322
|
attr_reader :item
|
308
323
|
attr_reader :value
|
324
|
+
attr_reader :queue
|
309
325
|
|
310
|
-
def initialize(item, queue_name = nil)
|
326
|
+
def initialize(item, queue_name = nil) # :nodoc:
|
311
327
|
@args = nil
|
312
328
|
@value = item
|
313
329
|
@item = item.is_a?(Hash) ? item : parse(item)
|
314
330
|
@queue = queue_name || @item["queue"]
|
315
331
|
end
|
316
332
|
|
317
|
-
def parse(item)
|
333
|
+
def parse(item) # :nodoc:
|
318
334
|
Sidekiq.load_json(item)
|
319
335
|
rescue JSON::ParserError
|
320
336
|
# If the job payload in Redis is invalid JSON, we'll load
|
@@ -354,27 +370,31 @@ module Sidekiq
|
|
354
370
|
def display_args
|
355
371
|
# Unwrap known wrappers so they show up in a human-friendly manner in the Web UI
|
356
372
|
@display_args ||= case klass
|
357
|
-
|
358
|
-
|
359
|
-
|
360
|
-
|
361
|
-
|
362
|
-
|
363
|
-
|
364
|
-
|
365
|
-
|
366
|
-
|
367
|
-
|
368
|
-
|
369
|
-
|
370
|
-
|
371
|
-
|
372
|
-
|
373
|
-
|
374
|
-
|
375
|
-
|
376
|
-
|
377
|
-
|
373
|
+
when /\ASidekiq::Extensions::Delayed/
|
374
|
+
safe_load(args[0], args) do |_, _, arg, kwarg|
|
375
|
+
if !kwarg || kwarg.empty?
|
376
|
+
arg
|
377
|
+
else
|
378
|
+
[arg, kwarg]
|
379
|
+
end
|
380
|
+
end
|
381
|
+
when "ActiveJob::QueueAdapters::SidekiqAdapter::JobWrapper"
|
382
|
+
job_args = self["wrapped"] ? args[0]["arguments"] : []
|
383
|
+
if (self["wrapped"] || args[0]) == "ActionMailer::DeliveryJob"
|
384
|
+
# remove MailerClass, mailer_method and 'deliver_now'
|
385
|
+
job_args.drop(3)
|
386
|
+
elsif (self["wrapped"] || args[0]) == "ActionMailer::MailDeliveryJob"
|
387
|
+
# remove MailerClass, mailer_method and 'deliver_now'
|
388
|
+
job_args.drop(3).first["args"]
|
389
|
+
else
|
390
|
+
job_args
|
391
|
+
end
|
392
|
+
else
|
393
|
+
if self["encrypt"]
|
394
|
+
# no point in showing 150+ bytes of random garbage
|
395
|
+
args[-1] = "[encrypted data]"
|
396
|
+
end
|
397
|
+
args
|
378
398
|
end
|
379
399
|
end
|
380
400
|
|
@@ -408,15 +428,12 @@ module Sidekiq
|
|
408
428
|
end
|
409
429
|
end
|
410
430
|
|
411
|
-
attr_reader :queue
|
412
|
-
|
413
431
|
def latency
|
414
432
|
now = Time.now.to_f
|
415
433
|
now - (@item["enqueued_at"] || @item["created_at"] || now)
|
416
434
|
end
|
417
435
|
|
418
|
-
|
419
|
-
# Remove this job from the queue.
|
436
|
+
# Remove this job from the queue
|
420
437
|
def delete
|
421
438
|
count = Sidekiq.redis { |conn|
|
422
439
|
conn.lrem("queue:#{@queue}", 1, @value)
|
@@ -424,6 +441,7 @@ module Sidekiq
|
|
424
441
|
count != 0
|
425
442
|
end
|
426
443
|
|
444
|
+
# Access arbitrary attributes within the job hash
|
427
445
|
def [](name)
|
428
446
|
# nil will happen if the JSON fails to parse.
|
429
447
|
# We don't guarantee Sidekiq will work with bad job JSON but we should
|
@@ -438,7 +456,8 @@ module Sidekiq
|
|
438
456
|
rescue => ex
|
439
457
|
# #1761 in dev mode, it's possible to have jobs enqueued which haven't been loaded into
|
440
458
|
# memory yet so the YAML can't be loaded.
|
441
|
-
|
459
|
+
# TODO is this still necessary? Zeitwerk reloader should handle?
|
460
|
+
Sidekiq.logger.warn "Unable to load YAML: #{ex.message}" unless Sidekiq.config[:environment] == "development"
|
442
461
|
default
|
443
462
|
end
|
444
463
|
|
@@ -460,13 +479,15 @@ module Sidekiq
|
|
460
479
|
end
|
461
480
|
end
|
462
481
|
|
482
|
+
# Represents a job within a Redis sorted set where the score
|
483
|
+
# represents a timestamp for the job.
|
463
484
|
class SortedEntry < JobRecord
|
464
485
|
attr_reader :score
|
465
486
|
attr_reader :parent
|
466
487
|
|
467
|
-
def initialize(parent, score, item)
|
488
|
+
def initialize(parent, score, item) # :nodoc:
|
468
489
|
super(item)
|
469
|
-
@score = score
|
490
|
+
@score = Float(score)
|
470
491
|
@parent = parent
|
471
492
|
end
|
472
493
|
|
@@ -482,12 +503,17 @@ module Sidekiq
|
|
482
503
|
end
|
483
504
|
end
|
484
505
|
|
506
|
+
# Change the scheduled time for this job.
|
507
|
+
#
|
508
|
+
# @param [Time] the new timestamp when this job will be enqueued.
|
485
509
|
def reschedule(at)
|
486
510
|
Sidekiq.redis do |conn|
|
487
511
|
conn.zincrby(@parent.name, at.to_f - @score, Sidekiq.dump_json(@item))
|
488
512
|
end
|
489
513
|
end
|
490
514
|
|
515
|
+
# Enqueue this job from the scheduled or dead set so it will
|
516
|
+
# be executed at some point in the near future.
|
491
517
|
def add_to_queue
|
492
518
|
remove_job do |message|
|
493
519
|
msg = Sidekiq.load_json(message)
|
@@ -495,6 +521,8 @@ module Sidekiq
|
|
495
521
|
end
|
496
522
|
end
|
497
523
|
|
524
|
+
# enqueue this job from the retry set so it will be executed
|
525
|
+
# at some point in the near future.
|
498
526
|
def retry
|
499
527
|
remove_job do |message|
|
500
528
|
msg = Sidekiq.load_json(message)
|
@@ -503,8 +531,7 @@ module Sidekiq
|
|
503
531
|
end
|
504
532
|
end
|
505
533
|
|
506
|
-
|
507
|
-
# Place job in the dead set
|
534
|
+
# Move this job from its current set into the Dead set.
|
508
535
|
def kill
|
509
536
|
remove_job do |message|
|
510
537
|
DeadSet.new.kill(message)
|
@@ -519,9 +546,9 @@ module Sidekiq
|
|
519
546
|
|
520
547
|
def remove_job
|
521
548
|
Sidekiq.redis do |conn|
|
522
|
-
results = conn.multi {
|
523
|
-
|
524
|
-
|
549
|
+
results = conn.multi { |transaction|
|
550
|
+
transaction.zrangebyscore(parent.name, score, score)
|
551
|
+
transaction.zremrangebyscore(parent.name, score, score)
|
525
552
|
}.first
|
526
553
|
|
527
554
|
if results.size == 1
|
@@ -542,9 +569,9 @@ module Sidekiq
|
|
542
569
|
yield msg if msg
|
543
570
|
|
544
571
|
# push the rest back onto the sorted set
|
545
|
-
conn.multi do
|
572
|
+
conn.multi do |transaction|
|
546
573
|
nonmatched.each do |message|
|
547
|
-
|
574
|
+
transaction.zadd(parent.name, score.to_f.to_s, message)
|
548
575
|
end
|
549
576
|
end
|
550
577
|
end
|
@@ -583,6 +610,10 @@ module Sidekiq
|
|
583
610
|
end
|
584
611
|
end
|
585
612
|
alias_method :💣, :clear
|
613
|
+
|
614
|
+
def as_json(options = nil) # :nodoc:
|
615
|
+
{name: name} # 5336
|
616
|
+
end
|
586
617
|
end
|
587
618
|
|
588
619
|
class JobSet < SortedSet
|
@@ -602,7 +633,7 @@ module Sidekiq
|
|
602
633
|
range_start = page * page_size + offset_size
|
603
634
|
range_end = range_start + page_size - 1
|
604
635
|
elements = Sidekiq.redis { |conn|
|
605
|
-
conn.zrange name, range_start, range_end,
|
636
|
+
conn.zrange name, range_start, range_end, withscores: true
|
606
637
|
}
|
607
638
|
break if elements.empty?
|
608
639
|
page -= 1
|
@@ -625,7 +656,7 @@ module Sidekiq
|
|
625
656
|
end
|
626
657
|
|
627
658
|
elements = Sidekiq.redis { |conn|
|
628
|
-
conn.zrangebyscore(name, begin_score, end_score,
|
659
|
+
conn.zrangebyscore(name, begin_score, end_score, withscores: true)
|
629
660
|
}
|
630
661
|
|
631
662
|
elements.each_with_object([]) do |element, result|
|
@@ -731,10 +762,10 @@ module Sidekiq
|
|
731
762
|
def kill(message, opts = {})
|
732
763
|
now = Time.now.to_f
|
733
764
|
Sidekiq.redis do |conn|
|
734
|
-
conn.multi do
|
735
|
-
|
736
|
-
|
737
|
-
|
765
|
+
conn.multi do |transaction|
|
766
|
+
transaction.zadd(name, now.to_s, message)
|
767
|
+
transaction.zremrangebyscore(name, "-inf", now - self.class.timeout)
|
768
|
+
transaction.zremrangebyrank(name, 0, - self.class.max_jobs)
|
738
769
|
end
|
739
770
|
end
|
740
771
|
|
@@ -754,11 +785,11 @@ module Sidekiq
|
|
754
785
|
end
|
755
786
|
|
756
787
|
def self.max_jobs
|
757
|
-
Sidekiq
|
788
|
+
Sidekiq[:dead_max_jobs]
|
758
789
|
end
|
759
790
|
|
760
791
|
def self.timeout
|
761
|
-
Sidekiq
|
792
|
+
Sidekiq[:dead_timeout_in_seconds]
|
762
793
|
end
|
763
794
|
end
|
764
795
|
|
@@ -782,9 +813,9 @@ module Sidekiq
|
|
782
813
|
count = 0
|
783
814
|
Sidekiq.redis do |conn|
|
784
815
|
procs = conn.sscan_each("processes").to_a.sort
|
785
|
-
heartbeats = conn.pipelined {
|
816
|
+
heartbeats = conn.pipelined { |pipeline|
|
786
817
|
procs.each do |key|
|
787
|
-
|
818
|
+
pipeline.hget(key, "info")
|
788
819
|
end
|
789
820
|
}
|
790
821
|
|
@@ -806,9 +837,9 @@ module Sidekiq
|
|
806
837
|
# We're making a tradeoff here between consuming more memory instead of
|
807
838
|
# making more roundtrips to Redis, but if you have hundreds or thousands of workers,
|
808
839
|
# you'll be happier this way
|
809
|
-
conn.pipelined do
|
840
|
+
conn.pipelined do |pipeline|
|
810
841
|
procs.each do |key|
|
811
|
-
|
842
|
+
pipeline.hmget(key, "info", "busy", "beat", "quiet", "rss", "rtt_us")
|
812
843
|
end
|
813
844
|
end
|
814
845
|
}
|
@@ -922,9 +953,9 @@ module Sidekiq
|
|
922
953
|
def signal(sig)
|
923
954
|
key = "#{identity}-signals"
|
924
955
|
Sidekiq.redis do |c|
|
925
|
-
c.multi do
|
926
|
-
|
927
|
-
|
956
|
+
c.multi do |transaction|
|
957
|
+
transaction.lpush(key, sig)
|
958
|
+
transaction.expire(key, 60)
|
928
959
|
end
|
929
960
|
end
|
930
961
|
end
|
@@ -958,9 +989,9 @@ module Sidekiq
|
|
958
989
|
Sidekiq.redis do |conn|
|
959
990
|
procs = conn.sscan_each("processes").to_a
|
960
991
|
procs.sort.each do |key|
|
961
|
-
valid, workers = conn.pipelined {
|
962
|
-
|
963
|
-
|
992
|
+
valid, workers = conn.pipelined { |pipeline|
|
993
|
+
pipeline.exists?(key)
|
994
|
+
pipeline.hgetall("#{key}:work")
|
964
995
|
}
|
965
996
|
next unless valid
|
966
997
|
workers.each_pair do |tid, json|
|
@@ -988,9 +1019,9 @@ module Sidekiq
|
|
988
1019
|
if procs.empty?
|
989
1020
|
0
|
990
1021
|
else
|
991
|
-
conn.pipelined {
|
1022
|
+
conn.pipelined { |pipeline|
|
992
1023
|
procs.each do |key|
|
993
|
-
|
1024
|
+
pipeline.hget(key, "busy")
|
994
1025
|
end
|
995
1026
|
}.sum(&:to_i)
|
996
1027
|
end
|
data/lib/sidekiq/cli.rb
CHANGED
@@ -9,18 +9,23 @@ require "erb"
|
|
9
9
|
require "fileutils"
|
10
10
|
|
11
11
|
require "sidekiq"
|
12
|
+
require "sidekiq/component"
|
12
13
|
require "sidekiq/launcher"
|
13
|
-
require "sidekiq/util"
|
14
14
|
|
15
|
-
module Sidekiq
|
15
|
+
module Sidekiq # :nodoc:
|
16
16
|
class CLI
|
17
|
-
include
|
17
|
+
include Sidekiq::Component
|
18
18
|
include Singleton unless $TESTING
|
19
19
|
|
20
20
|
attr_accessor :launcher
|
21
21
|
attr_accessor :environment
|
22
|
+
attr_accessor :config
|
23
|
+
|
24
|
+
def parse(args = ARGV.dup)
|
25
|
+
@config = Sidekiq
|
26
|
+
@config[:error_handlers].clear
|
27
|
+
@config[:error_handlers] << @config.method(:default_error_handler)
|
22
28
|
|
23
|
-
def parse(args = ARGV)
|
24
29
|
setup_options(args)
|
25
30
|
initialize_logger
|
26
31
|
validate!
|
@@ -36,7 +41,7 @@ module Sidekiq
|
|
36
41
|
def run(boot_app: true)
|
37
42
|
boot_application if boot_app
|
38
43
|
|
39
|
-
if environment == "development" && $stdout.tty? &&
|
44
|
+
if environment == "development" && $stdout.tty? && @config.log_formatter.is_a?(Sidekiq::Logger::Formatters::Pretty)
|
40
45
|
print_banner
|
41
46
|
end
|
42
47
|
logger.info "Booted Rails #{::Rails.version} application in #{environment} environment" if rails_app?
|
@@ -67,7 +72,7 @@ module Sidekiq
|
|
67
72
|
|
68
73
|
# touch the connection pool so it is created before we
|
69
74
|
# fire startup and start multithreading.
|
70
|
-
info =
|
75
|
+
info = @config.redis_info
|
71
76
|
ver = info["redis_version"]
|
72
77
|
raise "You are connecting to Redis v#{ver}, Sidekiq requires Redis v4.0.0 or greater" if ver < "4"
|
73
78
|
|
@@ -85,22 +90,22 @@ module Sidekiq
|
|
85
90
|
|
86
91
|
# Since the user can pass us a connection pool explicitly in the initializer, we
|
87
92
|
# need to verify the size is large enough or else Sidekiq's performance is dramatically slowed.
|
88
|
-
cursize =
|
89
|
-
needed =
|
93
|
+
cursize = @config.redis_pool.size
|
94
|
+
needed = @config[:concurrency] + 2
|
90
95
|
raise "Your pool of #{cursize} Redis connections is too small, please increase the size to at least #{needed}" if cursize < needed
|
91
96
|
|
92
97
|
# cache process identity
|
93
|
-
|
98
|
+
@config[:identity] = identity
|
94
99
|
|
95
100
|
# Touch middleware so it isn't lazy loaded by multiple threads, #3043
|
96
|
-
|
101
|
+
@config.server_middleware
|
97
102
|
|
98
103
|
# Before this point, the process is initializing with just the main thread.
|
99
104
|
# Starting here the process will now have multiple threads running.
|
100
105
|
fire_event(:startup, reverse: false, reraise: true)
|
101
106
|
|
102
|
-
logger.debug { "Client Middleware: #{
|
103
|
-
logger.debug { "Server Middleware: #{
|
107
|
+
logger.debug { "Client Middleware: #{@config.client_middleware.map(&:klass).join(", ")}" }
|
108
|
+
logger.debug { "Server Middleware: #{@config.server_middleware.map(&:klass).join(", ")}" }
|
104
109
|
|
105
110
|
launch(self_read)
|
106
111
|
end
|
@@ -110,13 +115,13 @@ module Sidekiq
|
|
110
115
|
logger.info "Starting processing, hit Ctrl-C to stop"
|
111
116
|
end
|
112
117
|
|
113
|
-
@launcher = Sidekiq::Launcher.new(
|
118
|
+
@launcher = Sidekiq::Launcher.new(@config)
|
114
119
|
|
115
120
|
begin
|
116
121
|
launcher.run
|
117
122
|
|
118
|
-
while
|
119
|
-
signal =
|
123
|
+
while self_read.wait_readable
|
124
|
+
signal = self_read.gets.strip
|
120
125
|
handle_signal(signal)
|
121
126
|
end
|
122
127
|
rescue Interrupt
|
@@ -173,25 +178,25 @@ module Sidekiq
|
|
173
178
|
# Heroku sends TERM and then waits 30 seconds for process to exit.
|
174
179
|
"TERM" => ->(cli) { raise Interrupt },
|
175
180
|
"TSTP" => ->(cli) {
|
176
|
-
|
181
|
+
cli.logger.info "Received TSTP, no longer accepting new work"
|
177
182
|
cli.launcher.quiet
|
178
183
|
},
|
179
184
|
"TTIN" => ->(cli) {
|
180
185
|
Thread.list.each do |thread|
|
181
|
-
|
186
|
+
cli.logger.warn "Thread TID-#{(thread.object_id ^ ::Process.pid).to_s(36)} #{thread.name}"
|
182
187
|
if thread.backtrace
|
183
|
-
|
188
|
+
cli.logger.warn thread.backtrace.join("\n")
|
184
189
|
else
|
185
|
-
|
190
|
+
cli.logger.warn "<no backtrace available>"
|
186
191
|
end
|
187
192
|
end
|
188
193
|
}
|
189
194
|
}
|
190
|
-
UNHANDLED_SIGNAL_HANDLER = ->(cli) {
|
195
|
+
UNHANDLED_SIGNAL_HANDLER = ->(cli) { cli.logger.info "No signal handler registered, ignoring" }
|
191
196
|
SIGNAL_HANDLERS.default = UNHANDLED_SIGNAL_HANDLER
|
192
197
|
|
193
198
|
def handle_signal(sig)
|
194
|
-
|
199
|
+
logger.debug "Got #{sig} signal"
|
195
200
|
SIGNAL_HANDLERS[sig].call(self)
|
196
201
|
end
|
197
202
|
|
@@ -237,7 +242,7 @@ module Sidekiq
|
|
237
242
|
config_dir = if File.directory?(opts[:require].to_s)
|
238
243
|
File.join(opts[:require], "config")
|
239
244
|
else
|
240
|
-
File.join(
|
245
|
+
File.join(@config[:require], "config")
|
241
246
|
end
|
242
247
|
|
243
248
|
%w[sidekiq.yml sidekiq.yml.erb].each do |config_file|
|
@@ -254,27 +259,23 @@ module Sidekiq
|
|
254
259
|
opts[:concurrency] = Integer(ENV["RAILS_MAX_THREADS"]) if opts[:concurrency].nil? && ENV["RAILS_MAX_THREADS"]
|
255
260
|
|
256
261
|
# merge with defaults
|
257
|
-
|
258
|
-
end
|
259
|
-
|
260
|
-
def options
|
261
|
-
Sidekiq.options
|
262
|
+
@config.merge!(opts)
|
262
263
|
end
|
263
264
|
|
264
265
|
def boot_application
|
265
266
|
ENV["RACK_ENV"] = ENV["RAILS_ENV"] = environment
|
266
267
|
|
267
|
-
if File.directory?(
|
268
|
+
if File.directory?(@config[:require])
|
268
269
|
require "rails"
|
269
270
|
if ::Rails::VERSION::MAJOR < 5
|
270
271
|
raise "Sidekiq no longer supports this version of Rails"
|
271
272
|
else
|
272
273
|
require "sidekiq/rails"
|
273
|
-
require File.expand_path("#{
|
274
|
+
require File.expand_path("#{@config[:require]}/config/environment.rb")
|
274
275
|
end
|
275
|
-
|
276
|
+
@config[:tag] ||= default_tag
|
276
277
|
else
|
277
|
-
require
|
278
|
+
require @config[:require]
|
278
279
|
end
|
279
280
|
end
|
280
281
|
|
@@ -291,18 +292,18 @@ module Sidekiq
|
|
291
292
|
end
|
292
293
|
|
293
294
|
def validate!
|
294
|
-
if !File.exist?(
|
295
|
-
(File.directory?(
|
295
|
+
if !File.exist?(@config[:require]) ||
|
296
|
+
(File.directory?(@config[:require]) && !File.exist?("#{@config[:require]}/config/application.rb"))
|
296
297
|
logger.info "=================================================================="
|
297
298
|
logger.info " Please point Sidekiq to a Rails application or a Ruby file "
|
298
|
-
logger.info " to load your
|
299
|
+
logger.info " to load your job classes with -r [DIR|FILE]."
|
299
300
|
logger.info "=================================================================="
|
300
301
|
logger.info @parser
|
301
302
|
die(1)
|
302
303
|
end
|
303
304
|
|
304
305
|
[:concurrency, :timeout].each do |opt|
|
305
|
-
raise ArgumentError, "#{opt}: #{
|
306
|
+
raise ArgumentError, "#{opt}: #{@config[opt]} is not a valid value" if @config[opt].to_i <= 0
|
306
307
|
end
|
307
308
|
end
|
308
309
|
|
@@ -336,7 +337,7 @@ module Sidekiq
|
|
336
337
|
parse_queue opts, queue, weight
|
337
338
|
end
|
338
339
|
|
339
|
-
o.on "-r", "--require [PATH|DIR]", "Location of Rails application with
|
340
|
+
o.on "-r", "--require [PATH|DIR]", "Location of Rails application with jobs or file to require" do |arg|
|
340
341
|
opts[:require] = arg
|
341
342
|
end
|
342
343
|
|
@@ -376,13 +377,13 @@ module Sidekiq
|
|
376
377
|
end
|
377
378
|
|
378
379
|
def initialize_logger
|
379
|
-
|
380
|
+
@config.logger.level = ::Logger::DEBUG if @config[:verbose]
|
380
381
|
end
|
381
382
|
|
382
383
|
def parse_config(path)
|
383
384
|
erb = ERB.new(File.read(path))
|
384
385
|
erb.filename = File.expand_path(path)
|
385
|
-
opts =
|
386
|
+
opts = load_yaml(erb.result) || {}
|
386
387
|
|
387
388
|
if opts.respond_to? :deep_symbolize_keys!
|
388
389
|
opts.deep_symbolize_keys!
|
@@ -398,6 +399,14 @@ module Sidekiq
|
|
398
399
|
opts
|
399
400
|
end
|
400
401
|
|
402
|
+
def load_yaml(src)
|
403
|
+
if Psych::VERSION > "4.0"
|
404
|
+
YAML.safe_load(src, permitted_classes: [Symbol], aliases: true)
|
405
|
+
else
|
406
|
+
YAML.load(src)
|
407
|
+
end
|
408
|
+
end
|
409
|
+
|
401
410
|
def parse_queues(opts, queues_and_weights)
|
402
411
|
queues_and_weights.each { |queue_and_weight| parse_queue(opts, *queue_and_weight) }
|
403
412
|
end
|