sidekiq-ultimate 0.0.1.alpha.16 → 0.0.1.alpha.17
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/lib/sidekiq/ultimate/expirable_set.rb +9 -16
- data/lib/sidekiq/ultimate/fetch.rb +23 -15
- data/lib/sidekiq/ultimate/fetch/debug.rb +22 -0
- data/lib/sidekiq/ultimate/version.rb +1 -1
- metadata +2 -2
- data/lib/sidekiq/ultimate/debugging.rb +0 -13
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: f6f524ab4f87ba7a8c687046e2ef2893b29a709d35765620df2c212bb5325cc2
|
4
|
+
data.tar.gz: 7be36332d763fb27106eb4212a9454aa54b665df630ec27df90c777fc0c7638b
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: ce2c291e7b34d21adb5c9d1c57fb48208eb385714a9650233519f507d878ecc6efbb6089eb284e149ce262ea5e1b04e5f586628d5143bb42d4617e3e0729f53c
|
7
|
+
data.tar.gz: f4aca94bb0798554e6fcfda6617af4c9ec6f905ea31dcdceff1c8fb450523f9c423060630cba0d870b5ef03ad9b1ccf9fb19dda1c8587bfe9f1c35cbad8480c0
|
@@ -4,8 +4,6 @@ require "monitor"
|
|
4
4
|
|
5
5
|
require "concurrent/utility/monotonic_time"
|
6
6
|
|
7
|
-
require "sidekiq/ultimate/debugging"
|
8
|
-
|
9
7
|
module Sidekiq
|
10
8
|
module Ultimate
|
11
9
|
# List that tracks when elements were added and enumerates over those not
|
@@ -33,7 +31,6 @@ module Sidekiq
|
|
33
31
|
# @private
|
34
32
|
class ExpirableSet
|
35
33
|
include Enumerable
|
36
|
-
include Debugging
|
37
34
|
|
38
35
|
# Create a new ExpirableSet instance.
|
39
36
|
def initialize
|
@@ -41,6 +38,11 @@ module Sidekiq
|
|
41
38
|
@mon = Monitor.new
|
42
39
|
end
|
43
40
|
|
41
|
+
# Allow implicit coercion to Array:
|
42
|
+
#
|
43
|
+
# ["x"] + ExpirableSet.new.add("y", :ttl => 10) # => ["x", "y"]
|
44
|
+
#
|
45
|
+
# @return [Array]
|
44
46
|
alias to_ary to_a
|
45
47
|
|
46
48
|
# Adds given element into the set.
|
@@ -53,15 +55,9 @@ module Sidekiq
|
|
53
55
|
expires_at = Concurrent.monotonic_time + ttl
|
54
56
|
|
55
57
|
# do not allow decrease element's expiry
|
56
|
-
if @set
|
57
|
-
|
58
|
-
|
59
|
-
"proposed expiry was: #{expires_at}"
|
60
|
-
end
|
61
|
-
else
|
62
|
-
@set[element] = expires_at
|
63
|
-
debug! { "#{element}'s expiry set to: #{expires_at}" }
|
64
|
-
end
|
58
|
+
break if @set.key?(element) && expires_at <= @set[element]
|
59
|
+
|
60
|
+
@set[element] = expires_at
|
65
61
|
end
|
66
62
|
|
67
63
|
self
|
@@ -78,10 +74,7 @@ module Sidekiq
|
|
78
74
|
|
79
75
|
@mon.synchronize do
|
80
76
|
horizon = Concurrent.monotonic_time
|
81
|
-
|
82
|
-
debug! { "Yielding elements above #{horizon} horizon" }
|
83
|
-
|
84
|
-
@set.each { |k, v| v < horizon ? @set.delete(k) : yield(k) }
|
77
|
+
@set.each { |k, v| yield(k) if horizon <= v }
|
85
78
|
end
|
86
79
|
|
87
80
|
self
|
@@ -2,7 +2,6 @@
|
|
2
2
|
|
3
3
|
require "sidekiq/throttled"
|
4
4
|
|
5
|
-
require "sidekiq/ultimate/debugging"
|
6
5
|
require "sidekiq/ultimate/expirable_set"
|
7
6
|
require "sidekiq/ultimate/queue_name"
|
8
7
|
require "sidekiq/ultimate/resurrector"
|
@@ -12,33 +11,32 @@ module Sidekiq
|
|
12
11
|
module Ultimate
|
13
12
|
# Throttled reliable fetcher implementing reliable queue pattern.
|
14
13
|
class Fetch
|
15
|
-
|
16
|
-
|
17
|
-
# Timeout to sleep between fetch retries in case of no job received.
|
14
|
+
# Delay between fetch retries in case of no job received.
|
18
15
|
TIMEOUT = 2
|
19
16
|
|
17
|
+
# Delay between queue poll attempts if last poll returned no jobs for it.
|
20
18
|
QUEUE_TIMEOUT = 5
|
21
19
|
|
22
|
-
#
|
23
|
-
# of it was throttled.
|
20
|
+
# Delay between queue poll attempts if it's last job was throttled.
|
24
21
|
THROTTLE_TIMEOUT = 15
|
25
22
|
|
26
23
|
def initialize(options)
|
27
24
|
@exhausted = ExpirableSet.new
|
28
25
|
|
26
|
+
@debug = ENV["DEBUG_SIDEKIQ_ULTIMATE"] ? {} : nil
|
29
27
|
@strict = options[:strict] ? true : false
|
30
28
|
@queues = options[:queues].map { |name| QueueName.new(name) }
|
31
29
|
|
32
30
|
@queues.uniq! if @strict
|
33
31
|
end
|
34
32
|
|
33
|
+
# @return [UnitOfWork] if work can be processed
|
35
34
|
def retrieve_work
|
36
35
|
work = retrieve
|
37
36
|
|
38
37
|
if work&.throttled?
|
39
38
|
work.requeue_throttled
|
40
39
|
|
41
|
-
debug! { "Queue #{work.queue} got throttled job." }
|
42
40
|
@exhausted.add(work.queue, :ttl => THROTTLE_TIMEOUT)
|
43
41
|
|
44
42
|
return nil
|
@@ -61,28 +59,26 @@ module Sidekiq
|
|
61
59
|
def retrieve
|
62
60
|
Sidekiq.redis do |redis|
|
63
61
|
queues.each do |queue|
|
62
|
+
debug!(queue)
|
63
|
+
|
64
64
|
job = redis.rpoplpush(queue.pending, queue.inproc)
|
65
65
|
return UnitOfWork.new(queue, job) if job
|
66
66
|
|
67
|
-
debug! { "Queue #{queue} has no job." }
|
68
67
|
@exhausted.add(queue, :ttl => QUEUE_TIMEOUT)
|
69
68
|
end
|
70
69
|
end
|
71
70
|
|
72
|
-
debug! { "No jobs in any queues." }
|
73
|
-
|
74
71
|
sleep TIMEOUT
|
75
72
|
nil
|
76
73
|
end
|
77
74
|
|
78
75
|
def queues
|
79
|
-
queues
|
80
|
-
queues -= @exhausted_queues.to_a
|
81
|
-
queues -= paused_queues unless queues.empty?
|
76
|
+
queues = (@strict ? @queues : @queues.shuffle.uniq) - @exhausted.to_a
|
82
77
|
|
83
|
-
|
78
|
+
# Avoid calling heavier `paused_queue` if there's nothing to filter out
|
79
|
+
return queues if queues.empty?
|
84
80
|
|
85
|
-
queues
|
81
|
+
queues - paused_queues
|
86
82
|
end
|
87
83
|
|
88
84
|
def paused_queues
|
@@ -90,6 +86,18 @@ module Sidekiq
|
|
90
86
|
instance_variable_get(:@paused_queues).
|
91
87
|
map { |q| QueueName[q] }
|
92
88
|
end
|
89
|
+
|
90
|
+
def debug!(queue)
|
91
|
+
return unless @debug
|
92
|
+
|
93
|
+
previous, @debug[queue] = @debug[queue], Concurrent.monotonic_time
|
94
|
+
|
95
|
+
return unless previous
|
96
|
+
|
97
|
+
Sidekiq.logger.debug do
|
98
|
+
"Queue #{queue} last time polled: #{@debug[queue] - previous} s ago"
|
99
|
+
end
|
100
|
+
end
|
93
101
|
end
|
94
102
|
end
|
95
103
|
end
|
@@ -0,0 +1,22 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require "sidekiq/throttled"
|
4
|
+
|
5
|
+
require "sidekiq/ultimate/debugging"
|
6
|
+
require "sidekiq/ultimate/expirable_set"
|
7
|
+
require "sidekiq/ultimate/queue_name"
|
8
|
+
require "sidekiq/ultimate/resurrector"
|
9
|
+
require "sidekiq/ultimate/unit_of_work"
|
10
|
+
|
11
|
+
module Sidekiq
|
12
|
+
module Ultimate
|
13
|
+
# Throttled reliable fetcher implementing reliable queue pattern.
|
14
|
+
class Fetch
|
15
|
+
class Debug
|
16
|
+
def initialize
|
17
|
+
@queues_stats = {}
|
18
|
+
end
|
19
|
+
end
|
20
|
+
end
|
21
|
+
end
|
22
|
+
end
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: sidekiq-ultimate
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.0.1.alpha.
|
4
|
+
version: 0.0.1.alpha.17
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Alexey Zapparov
|
@@ -112,9 +112,9 @@ files:
|
|
112
112
|
- README.md
|
113
113
|
- Rakefile
|
114
114
|
- lib/sidekiq/ultimate.rb
|
115
|
-
- lib/sidekiq/ultimate/debugging.rb
|
116
115
|
- lib/sidekiq/ultimate/expirable_set.rb
|
117
116
|
- lib/sidekiq/ultimate/fetch.rb
|
117
|
+
- lib/sidekiq/ultimate/fetch/debug.rb
|
118
118
|
- lib/sidekiq/ultimate/queue_name.rb
|
119
119
|
- lib/sidekiq/ultimate/resurrector.rb
|
120
120
|
- lib/sidekiq/ultimate/resurrector/resurrect.lua
|