sidekiq-limit_fetch 1.6 → 1.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/lib/sidekiq/extensions/queue.rb +9 -12
- data/lib/sidekiq/limit_fetch.rb +1 -3
- data/lib/sidekiq/limit_fetch/global/monitor.rb +4 -2
- data/lib/sidekiq/limit_fetch/global/selector.rb +5 -5
- data/lib/sidekiq/limit_fetch/global/semaphore.rb +13 -1
- data/lib/sidekiq/limit_fetch/queues.rb +4 -6
- data/lib/sidekiq/limit_fetch/unit_of_work.rb +6 -0
- data/sidekiq-limit_fetch.gemspec +1 -1
- data/spec/sidekiq/extensions/queue_spec.rb +45 -57
- data/spec/sidekiq/limit_fetch/global/monitor_spec.rb +2 -13
- data/spec/sidekiq/limit_fetch/queues_spec.rb +44 -69
- data/spec/sidekiq/limit_fetch/semaphore_spec.rb +35 -54
- data/spec/sidekiq/limit_fetch_spec.rb +23 -23
- data/spec/spec_helper.rb +7 -7
- metadata +1 -3
- data/lib/sidekiq/limit_fetch/local/selector.rb +0 -31
- data/lib/sidekiq/limit_fetch/local/semaphore.rb +0 -61
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 4c5121de39e03363b90b64633028f65ed5cf6983
|
4
|
+
data.tar.gz: 935a28cd47181070776e71fe60db08165cab2ffe
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 09f48f0e5dce160c809b4fdad660f35c0fe2ac27ee3f6d850d094fc2e995adc70c3915cefb2ff4bc3a3aea2785c4f2a5edf3ca65c0692aea923dfbcd8ccdb2f4
|
7
|
+
data.tar.gz: a4d226be6b1df6fad9ab438dfd60e8e29235c89540f209c3399c5dd3ef24781184e653af3ae7cdbc67d08104f3a8fa16d6a44084ca36ecf3780d14e024579d78
|
@@ -3,20 +3,17 @@ module Sidekiq
|
|
3
3
|
extend LimitFetch::Singleton, Forwardable
|
4
4
|
|
5
5
|
def_delegators :lock,
|
6
|
-
:limit,
|
7
|
-
:acquire,
|
8
|
-
:pause,
|
9
|
-
:block,
|
10
|
-
:paused?,
|
11
|
-
:unblocked,
|
12
|
-
:busy
|
6
|
+
:limit, :limit=,
|
7
|
+
:acquire, :release,
|
8
|
+
:pause, :unpause,
|
9
|
+
:block, :unblock,
|
10
|
+
:paused?, :blocking?,
|
11
|
+
:unblocked, :block_except,
|
12
|
+
:probed, :busy,
|
13
|
+
:increase_busy, :decrease_busy
|
13
14
|
|
14
15
|
def lock
|
15
|
-
@lock ||=
|
16
|
-
end
|
17
|
-
|
18
|
-
def mode
|
19
|
-
Sidekiq.options[:local] ? LimitFetch::Local : LimitFetch::Global
|
16
|
+
@lock ||= LimitFetch::Global::Semaphore.new name
|
20
17
|
end
|
21
18
|
end
|
22
19
|
end
|
data/lib/sidekiq/limit_fetch.rb
CHANGED
@@ -6,8 +6,6 @@ class Sidekiq::LimitFetch
|
|
6
6
|
|
7
7
|
require_relative 'limit_fetch/singleton'
|
8
8
|
require_relative 'limit_fetch/queues'
|
9
|
-
require_relative 'limit_fetch/local/semaphore'
|
10
|
-
require_relative 'limit_fetch/local/selector'
|
11
9
|
require_relative 'limit_fetch/global/semaphore'
|
12
10
|
require_relative 'limit_fetch/global/selector'
|
13
11
|
require_relative 'limit_fetch/global/monitor'
|
@@ -20,7 +18,7 @@ class Sidekiq::LimitFetch
|
|
20
18
|
end
|
21
19
|
|
22
20
|
def initialize(options)
|
23
|
-
Global::Monitor.start!
|
21
|
+
Global::Monitor.start!
|
24
22
|
@queues = Queues.new options
|
25
23
|
end
|
26
24
|
|
@@ -37,8 +37,10 @@ module Sidekiq::LimitFetch::Global
|
|
37
37
|
next if it.get heartbeat_key processor
|
38
38
|
|
39
39
|
it.del processor_key processor
|
40
|
-
|
41
|
-
it.
|
40
|
+
%w(limit_fetch:probed:* limit_fetch:busy:*).each do |pattern|
|
41
|
+
it.keys(pattern).each do |queue|
|
42
|
+
it.lrem queue, 0, processor
|
43
|
+
end
|
42
44
|
end
|
43
45
|
end
|
44
46
|
end
|
@@ -60,7 +60,7 @@ module Sidekiq::LimitFetch::Global
|
|
60
60
|
|
61
61
|
for _, queue in ipairs(queues) do
|
62
62
|
if not blocking_mode or unblocked[queue] then
|
63
|
-
local
|
63
|
+
local probed_key = namespace..'probed:'..queue
|
64
64
|
local pause_key = namespace..'pause:'..queue
|
65
65
|
local paused = redis.call('get', pause_key)
|
66
66
|
|
@@ -72,7 +72,7 @@ module Sidekiq::LimitFetch::Global
|
|
72
72
|
local can_block = redis.call('get', block_key)
|
73
73
|
|
74
74
|
if can_block or queue_limit then
|
75
|
-
queue_locks = redis.call('llen',
|
75
|
+
queue_locks = redis.call('llen', probed_key)
|
76
76
|
end
|
77
77
|
|
78
78
|
blocking_mode = can_block and queue_locks > 0
|
@@ -84,7 +84,7 @@ module Sidekiq::LimitFetch::Global
|
|
84
84
|
end
|
85
85
|
|
86
86
|
if not queue_limit or queue_limit > queue_locks then
|
87
|
-
redis.call('rpush',
|
87
|
+
redis.call('rpush', probed_key, worker_name)
|
88
88
|
table.insert(available, queue)
|
89
89
|
end
|
90
90
|
end
|
@@ -102,8 +102,8 @@ module Sidekiq::LimitFetch::Global
|
|
102
102
|
local queues = ARGV
|
103
103
|
|
104
104
|
for _, queue in ipairs(queues) do
|
105
|
-
local
|
106
|
-
redis.call('lrem',
|
105
|
+
local probed_key = namespace..'probed:'..queue
|
106
|
+
redis.call('lrem', probed_key, 1, worker_name)
|
107
107
|
end
|
108
108
|
LUA
|
109
109
|
end
|
@@ -23,13 +23,25 @@ module Sidekiq::LimitFetch::Global
|
|
23
23
|
end
|
24
24
|
|
25
25
|
def release
|
26
|
-
Selector.
|
26
|
+
redis {|it| it.lrem "#{PREFIX}:probed:#@name", 1, Selector.uuid }
|
27
27
|
end
|
28
28
|
|
29
29
|
def busy
|
30
30
|
redis {|it| it.llen "#{PREFIX}:busy:#@name" }
|
31
31
|
end
|
32
32
|
|
33
|
+
def increase_busy
|
34
|
+
redis {|it| it.rpush "#{PREFIX}:busy:#@name", Selector.uuid }
|
35
|
+
end
|
36
|
+
|
37
|
+
def decrease_busy
|
38
|
+
redis {|it| it.lrem "#{PREFIX}:busy:#@name", 1, Selector.uuid }
|
39
|
+
end
|
40
|
+
|
41
|
+
def probed
|
42
|
+
redis {|it| it.llen "#{PREFIX}:probed:#@name" }
|
43
|
+
end
|
44
|
+
|
33
45
|
def pause
|
34
46
|
redis {|it| it.set "#{PREFIX}:pause:#@name", true }
|
35
47
|
end
|
@@ -1,19 +1,17 @@
|
|
1
1
|
class Sidekiq::LimitFetch
|
2
2
|
class Queues
|
3
3
|
THREAD_KEY = :acquired_queues
|
4
|
-
attr_reader :selector
|
5
4
|
|
6
5
|
def initialize(options)
|
7
6
|
@queues = options[:queues]
|
8
7
|
options[:strict] ? strict_order! : weighted_order!
|
9
8
|
|
10
|
-
set_selector options[:local]
|
11
9
|
set_limits options[:limits]
|
12
10
|
set_blocks options[:blocking]
|
13
11
|
end
|
14
12
|
|
15
13
|
def acquire
|
16
|
-
|
14
|
+
selector.acquire(ordered_queues)
|
17
15
|
.tap {|it| save it }
|
18
16
|
.map {|it| "queue:#{it}" }
|
19
17
|
end
|
@@ -21,13 +19,13 @@ class Sidekiq::LimitFetch
|
|
21
19
|
def release_except(full_name)
|
22
20
|
queues = restore
|
23
21
|
queues.delete full_name[/queue:(.*)/, 1] if full_name
|
24
|
-
|
22
|
+
selector.release queues
|
25
23
|
end
|
26
24
|
|
27
25
|
private
|
28
26
|
|
29
|
-
def
|
30
|
-
|
27
|
+
def selector
|
28
|
+
Global::Selector
|
31
29
|
end
|
32
30
|
|
33
31
|
def set_limits(limits)
|
data/sidekiq-limit_fetch.gemspec
CHANGED
@@ -28,73 +28,61 @@ describe Sidekiq::Queue do
|
|
28
28
|
let(:name) { 'example' }
|
29
29
|
let(:queue) { Sidekiq::Queue[name] }
|
30
30
|
|
31
|
-
|
32
|
-
|
33
|
-
|
34
|
-
end
|
35
|
-
|
36
|
-
it 'should be pausable' do
|
37
|
-
queue.pause
|
38
|
-
queue.acquire.should_not be
|
39
|
-
end
|
40
|
-
|
41
|
-
it 'should be continuable' do
|
42
|
-
queue.pause
|
43
|
-
queue.unpause
|
44
|
-
queue.acquire.should be
|
45
|
-
end
|
31
|
+
it 'should be available' do
|
32
|
+
queue.acquire.should be
|
33
|
+
end
|
46
34
|
|
47
|
-
|
48
|
-
|
49
|
-
|
50
|
-
|
51
|
-
end
|
35
|
+
it 'should be pausable' do
|
36
|
+
queue.pause
|
37
|
+
queue.acquire.should_not be
|
38
|
+
end
|
52
39
|
|
53
|
-
|
54
|
-
|
55
|
-
|
56
|
-
|
57
|
-
|
58
|
-
end
|
40
|
+
it 'should be continuable' do
|
41
|
+
queue.pause
|
42
|
+
queue.unpause
|
43
|
+
queue.acquire.should be
|
44
|
+
end
|
59
45
|
|
60
|
-
|
61
|
-
|
62
|
-
|
63
|
-
|
64
|
-
|
46
|
+
it 'should be limitable' do
|
47
|
+
queue.limit = 1
|
48
|
+
queue.acquire.should be
|
49
|
+
queue.acquire.should_not be
|
50
|
+
end
|
65
51
|
|
66
|
-
|
67
|
-
|
68
|
-
|
69
|
-
|
70
|
-
|
71
|
-
|
52
|
+
it 'should be resizable' do
|
53
|
+
queue.limit = 0
|
54
|
+
queue.acquire.should_not be
|
55
|
+
queue.limit = nil
|
56
|
+
queue.acquire.should be
|
57
|
+
end
|
72
58
|
|
73
|
-
|
74
|
-
|
75
|
-
|
76
|
-
|
77
|
-
|
78
|
-
queue.should_not be_paused
|
79
|
-
end
|
59
|
+
it 'should be countable' do
|
60
|
+
queue.limit = 3
|
61
|
+
5.times { queue.acquire }
|
62
|
+
queue.probed.should == 3
|
63
|
+
end
|
80
64
|
|
81
|
-
|
82
|
-
|
83
|
-
|
84
|
-
|
85
|
-
|
86
|
-
queue.should_not be_blocking
|
87
|
-
end
|
65
|
+
it 'should be releasable' do
|
66
|
+
queue.acquire
|
67
|
+
queue.probed.should == 1
|
68
|
+
queue.release
|
69
|
+
queue.probed.should == 0
|
88
70
|
end
|
89
71
|
|
90
|
-
|
91
|
-
|
92
|
-
|
72
|
+
it 'should tell if paused' do
|
73
|
+
queue.should_not be_paused
|
74
|
+
queue.pause
|
75
|
+
queue.should be_paused
|
76
|
+
queue.unpause
|
77
|
+
queue.should_not be_paused
|
93
78
|
end
|
94
79
|
|
95
|
-
|
96
|
-
|
97
|
-
|
80
|
+
it 'should tell if blocking' do
|
81
|
+
queue.should_not be_blocking
|
82
|
+
queue.block
|
83
|
+
queue.should be_blocking
|
84
|
+
queue.unblock
|
85
|
+
queue.should_not be_blocking
|
98
86
|
end
|
99
87
|
end
|
100
88
|
end
|
@@ -7,17 +7,6 @@ describe Sidekiq::LimitFetch::Global::Monitor do
|
|
7
7
|
let(:name) { 'default' }
|
8
8
|
|
9
9
|
before :each do
|
10
|
-
# namespaces = [
|
11
|
-
# described_class::PROCESSOR_NAMESPACE,
|
12
|
-
# described_class::HEARTBEAT_NAMESPACE
|
13
|
-
# ]
|
14
|
-
|
15
|
-
# Sidekiq.redis do |it|
|
16
|
-
# namespaces.flat_map {|namespace|
|
17
|
-
# it.keys(namespace + '*')
|
18
|
-
# }.each {|key| it.del key }
|
19
|
-
# end
|
20
|
-
|
21
10
|
monitor
|
22
11
|
end
|
23
12
|
|
@@ -31,11 +20,11 @@ describe Sidekiq::LimitFetch::Global::Monitor do
|
|
31
20
|
it 'should remove invalidated old locks' do
|
32
21
|
2.times { queue.acquire }
|
33
22
|
sleep 2*ttl
|
34
|
-
queue.
|
23
|
+
queue.probed.should == 2
|
35
24
|
|
36
25
|
described_class.stub :update_heartbeat
|
37
26
|
sleep 2*ttl
|
38
|
-
queue.
|
27
|
+
queue.probed.should == 0
|
39
28
|
end
|
40
29
|
end
|
41
30
|
end
|
@@ -6,97 +6,72 @@ describe Sidekiq::LimitFetch::Queues do
|
|
6
6
|
let(:queues) { %w[queue1 queue2] }
|
7
7
|
let(:limits) {{ 'queue1' => 3 }}
|
8
8
|
let(:strict) { true }
|
9
|
-
let(:local) {}
|
10
9
|
let(:blocking) {}
|
11
10
|
|
12
11
|
let(:options) do
|
13
12
|
{ queues: queues,
|
14
13
|
limits: limits,
|
15
14
|
strict: strict,
|
16
|
-
local: local,
|
17
15
|
blocking: blocking }
|
18
16
|
end
|
19
17
|
|
20
|
-
|
21
|
-
|
18
|
+
it 'should acquire queues' do
|
19
|
+
subject.acquire
|
20
|
+
Sidekiq::Queue['queue1'].probed.should == 1
|
21
|
+
Sidekiq::Queue['queue2'].probed.should == 1
|
22
22
|
end
|
23
23
|
|
24
|
-
|
25
|
-
|
26
|
-
|
27
|
-
|
28
|
-
Sidekiq::Queue['queue2'].busy.should == 1
|
29
|
-
end
|
30
|
-
|
31
|
-
it 'should acquire dynamically blocking queues' do
|
32
|
-
subject.acquire
|
33
|
-
Sidekiq::Queue['queue1'].busy.should == 1
|
34
|
-
Sidekiq::Queue['queue2'].busy.should == 1
|
35
|
-
|
36
|
-
Sidekiq::Queue['queue1'].block
|
37
|
-
|
38
|
-
subject.acquire
|
39
|
-
Sidekiq::Queue['queue1'].busy.should == 2
|
40
|
-
Sidekiq::Queue['queue2'].busy.should == 1
|
41
|
-
end
|
42
|
-
|
43
|
-
it 'should block except given queues' do
|
44
|
-
Sidekiq::Queue['queue1'].block_except 'queue2'
|
45
|
-
subject.acquire
|
46
|
-
Sidekiq::Queue['queue1'].busy.should == 1
|
47
|
-
Sidekiq::Queue['queue2'].busy.should == 1
|
48
|
-
|
49
|
-
Sidekiq::Queue['queue1'].block_except 'queue404'
|
50
|
-
subject.acquire
|
51
|
-
Sidekiq::Queue['queue1'].busy.should == 2
|
52
|
-
Sidekiq::Queue['queue2'].busy.should == 1
|
53
|
-
end
|
24
|
+
it 'should acquire dynamically blocking queues' do
|
25
|
+
subject.acquire
|
26
|
+
Sidekiq::Queue['queue1'].probed.should == 1
|
27
|
+
Sidekiq::Queue['queue2'].probed.should == 1
|
54
28
|
|
55
|
-
|
56
|
-
subject.acquire
|
57
|
-
subject.release_except nil
|
58
|
-
Sidekiq::Queue['queue1'].busy.should == 0
|
59
|
-
Sidekiq::Queue['queue2'].busy.should == 0
|
60
|
-
end
|
29
|
+
Sidekiq::Queue['queue1'].block
|
61
30
|
|
62
|
-
|
63
|
-
|
64
|
-
|
65
|
-
|
66
|
-
Sidekiq::Queue['queue2'].busy.should == 0
|
67
|
-
end
|
31
|
+
subject.acquire
|
32
|
+
Sidekiq::Queue['queue1'].probed.should == 2
|
33
|
+
Sidekiq::Queue['queue2'].probed.should == 1
|
34
|
+
end
|
68
35
|
|
69
|
-
|
70
|
-
|
71
|
-
|
72
|
-
|
73
|
-
|
36
|
+
it 'should block except given queues' do
|
37
|
+
Sidekiq::Queue['queue1'].block_except 'queue2'
|
38
|
+
subject.acquire
|
39
|
+
Sidekiq::Queue['queue1'].probed.should == 1
|
40
|
+
Sidekiq::Queue['queue2'].probed.should == 1
|
74
41
|
|
75
|
-
|
76
|
-
|
42
|
+
Sidekiq::Queue['queue1'].block_except 'queue404'
|
43
|
+
subject.acquire
|
44
|
+
Sidekiq::Queue['queue1'].probed.should == 2
|
45
|
+
Sidekiq::Queue['queue2'].probed.should == 1
|
46
|
+
end
|
77
47
|
|
78
|
-
|
79
|
-
|
80
|
-
|
81
|
-
|
82
|
-
|
83
|
-
end
|
48
|
+
it 'should release queues' do
|
49
|
+
subject.acquire
|
50
|
+
subject.release_except nil
|
51
|
+
Sidekiq::Queue['queue1'].probed.should == 0
|
52
|
+
Sidekiq::Queue['queue2'].probed.should == 0
|
84
53
|
end
|
85
54
|
|
86
|
-
|
87
|
-
|
55
|
+
it 'should release queues except selected' do
|
56
|
+
subject.acquire
|
57
|
+
subject.release_except 'queue:queue1'
|
58
|
+
Sidekiq::Queue['queue1'].probed.should == 1
|
59
|
+
Sidekiq::Queue['queue2'].probed.should == 0
|
60
|
+
end
|
88
61
|
|
89
|
-
|
90
|
-
|
91
|
-
|
62
|
+
it 'should release when no queues was acquired' do
|
63
|
+
queues.each {|name| Sidekiq::Queue[name].pause }
|
64
|
+
subject.acquire
|
65
|
+
-> { subject.release_except nil }.should_not raise_exception
|
92
66
|
end
|
93
67
|
|
94
|
-
context '
|
95
|
-
let(:
|
96
|
-
it_should_behave_like :selector
|
68
|
+
context 'blocking' do
|
69
|
+
let(:blocking) { %w(queue1) }
|
97
70
|
|
98
|
-
it 'should
|
99
|
-
subject.
|
71
|
+
it 'should acquire blocking queues' do
|
72
|
+
3.times { subject.acquire }
|
73
|
+
Sidekiq::Queue['queue1'].probed.should == 3
|
74
|
+
Sidekiq::Queue['queue2'].probed.should == 1
|
100
75
|
end
|
101
76
|
end
|
102
77
|
|
@@ -1,68 +1,49 @@
|
|
1
1
|
require 'spec_helper'
|
2
2
|
|
3
3
|
describe 'semaphore' do
|
4
|
-
|
5
|
-
|
6
|
-
subject.limit.should_not be
|
7
|
-
end
|
8
|
-
|
9
|
-
it 'should set limit' do
|
10
|
-
subject.limit = 4
|
11
|
-
subject.limit.should == 4
|
12
|
-
end
|
13
|
-
|
14
|
-
it 'should acquire and count active tasks' do
|
15
|
-
3.times { subject.acquire }
|
16
|
-
subject.busy.should == 3
|
17
|
-
end
|
18
|
-
|
19
|
-
it 'should acquire tasks with regard to limit' do
|
20
|
-
subject.limit = 4
|
21
|
-
6.times { subject.acquire }
|
22
|
-
subject.busy.should == 4
|
23
|
-
end
|
4
|
+
let(:name) { 'default' }
|
5
|
+
subject { Sidekiq::LimitFetch::Global::Semaphore.new name }
|
24
6
|
|
25
|
-
|
26
|
-
|
27
|
-
|
28
|
-
subject.busy.should == 3
|
29
|
-
end
|
7
|
+
it 'should have no limit by default' do
|
8
|
+
subject.limit.should_not be
|
9
|
+
end
|
30
10
|
|
31
|
-
|
32
|
-
|
33
|
-
|
34
|
-
|
35
|
-
subject.busy.should == 3
|
36
|
-
2.times { subject.release }
|
37
|
-
subject.busy.should == 1
|
38
|
-
end
|
11
|
+
it 'should set limit' do
|
12
|
+
subject.limit = 4
|
13
|
+
subject.limit.should == 4
|
14
|
+
end
|
39
15
|
|
40
|
-
|
41
|
-
|
42
|
-
|
43
|
-
subject.unpause
|
44
|
-
2.times { subject.acquire }
|
45
|
-
subject.busy.should == 2
|
46
|
-
end
|
16
|
+
it 'should acquire and count active tasks' do
|
17
|
+
3.times { subject.acquire }
|
18
|
+
subject.probed.should == 3
|
47
19
|
end
|
48
20
|
|
49
|
-
|
21
|
+
it 'should acquire tasks with regard to limit' do
|
22
|
+
subject.limit = 4
|
23
|
+
6.times { subject.acquire }
|
24
|
+
subject.probed.should == 4
|
25
|
+
end
|
50
26
|
|
51
|
-
|
52
|
-
|
53
|
-
|
27
|
+
it 'should release active tasks' do
|
28
|
+
6.times { subject.acquire }
|
29
|
+
3.times { subject.release }
|
30
|
+
subject.probed.should == 3
|
54
31
|
end
|
55
32
|
|
56
|
-
|
57
|
-
|
58
|
-
|
33
|
+
it 'should pause tasks' do
|
34
|
+
3.times { subject.acquire }
|
35
|
+
subject.pause
|
36
|
+
2.times { subject.acquire }
|
37
|
+
subject.probed.should == 3
|
38
|
+
2.times { subject.release }
|
39
|
+
subject.probed.should == 1
|
40
|
+
end
|
59
41
|
|
60
|
-
|
61
|
-
|
62
|
-
|
63
|
-
|
64
|
-
|
65
|
-
|
66
|
-
end
|
42
|
+
it 'should unpause tasks' do
|
43
|
+
subject.pause
|
44
|
+
3.times { subject.acquire }
|
45
|
+
subject.unpause
|
46
|
+
2.times { subject.acquire }
|
47
|
+
subject.probed.should == 2
|
67
48
|
end
|
68
49
|
end
|
@@ -11,37 +11,37 @@ describe Sidekiq::LimitFetch do
|
|
11
11
|
end
|
12
12
|
|
13
13
|
subject { described_class.new options }
|
14
|
-
let(:options) {{ queues: queues, limits: limits
|
14
|
+
let(:options) {{ queues: queues, limits: limits }}
|
15
15
|
let(:queues) { %w(queue1 queue1 queue2 queue2) }
|
16
16
|
let(:limits) {{ 'queue1' => 1, 'queue2' => 2 }}
|
17
17
|
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
work.message.should == 'task1'
|
18
|
+
it 'should acquire lock on queue for execution' do
|
19
|
+
work = subject.retrieve_work
|
20
|
+
work.queue_name.should == 'queue1'
|
21
|
+
work.message.should == 'task1'
|
23
22
|
|
24
|
-
|
25
|
-
|
23
|
+
Sidekiq::Queue['queue1'].busy.should == 1
|
24
|
+
Sidekiq::Queue['queue2'].busy.should == 0
|
26
25
|
|
27
|
-
|
28
|
-
|
26
|
+
subject.retrieve_work.should_not be
|
27
|
+
work.requeue
|
29
28
|
|
30
|
-
|
31
|
-
|
29
|
+
Sidekiq::Queue['queue1'].busy.should == 0
|
30
|
+
Sidekiq::Queue['queue2'].busy.should == 0
|
32
31
|
|
33
|
-
|
34
|
-
|
35
|
-
end
|
36
|
-
end
|
32
|
+
work = subject.retrieve_work
|
33
|
+
work.message.should == 'task2'
|
37
34
|
|
38
|
-
|
39
|
-
|
40
|
-
|
41
|
-
|
35
|
+
Sidekiq::Queue['queue1'].busy.should == 1
|
36
|
+
Sidekiq::Queue['queue2'].busy.should == 0
|
37
|
+
|
38
|
+
subject.retrieve_work.should_not be
|
39
|
+
work.acknowledge
|
40
|
+
|
41
|
+
Sidekiq::Queue['queue1'].busy.should == 0
|
42
|
+
Sidekiq::Queue['queue2'].busy.should == 0
|
42
43
|
|
43
|
-
|
44
|
-
|
45
|
-
it_behaves_like :strategy
|
44
|
+
work = subject.retrieve_work
|
45
|
+
work.message.should == 'task1'
|
46
46
|
end
|
47
47
|
end
|
data/spec/spec_helper.rb
CHANGED
@@ -4,15 +4,15 @@ require 'sidekiq/fetch'
|
|
4
4
|
|
5
5
|
RSpec.configure do |config|
|
6
6
|
config.before :each do
|
7
|
-
Sidekiq::Queue.instance_variable_set :@instances, {}
|
8
|
-
Sidekiq.options[:local] = defined?(local) ? local : nil
|
9
|
-
|
10
7
|
Sidekiq.redis do |it|
|
11
8
|
clean_redis = ->(queue) do
|
12
|
-
it.
|
13
|
-
|
14
|
-
|
15
|
-
|
9
|
+
it.pipelined do
|
10
|
+
it.del "limit_fetch:limit:#{queue}"
|
11
|
+
it.del "limit_fetch:busy:#{queue}"
|
12
|
+
it.del "limit_fetch:probed:#{queue}"
|
13
|
+
it.del "limit_fetch:pause:#{queue}"
|
14
|
+
it.del "limit_fetch:block:#{queue}"
|
15
|
+
end
|
16
16
|
end
|
17
17
|
|
18
18
|
clean_redis.call(name) if defined?(name)
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: sidekiq-limit_fetch
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: '1.
|
4
|
+
version: '1.7'
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- brainopia
|
@@ -77,8 +77,6 @@ files:
|
|
77
77
|
- lib/sidekiq/limit_fetch/global/monitor.rb
|
78
78
|
- lib/sidekiq/limit_fetch/global/selector.rb
|
79
79
|
- lib/sidekiq/limit_fetch/global/semaphore.rb
|
80
|
-
- lib/sidekiq/limit_fetch/local/selector.rb
|
81
|
-
- lib/sidekiq/limit_fetch/local/semaphore.rb
|
82
80
|
- lib/sidekiq/limit_fetch/queues.rb
|
83
81
|
- lib/sidekiq/limit_fetch/singleton.rb
|
84
82
|
- lib/sidekiq/limit_fetch/unit_of_work.rb
|
@@ -1,31 +0,0 @@
|
|
1
|
-
module Sidekiq::LimitFetch::Local
|
2
|
-
module Selector
|
3
|
-
extend self
|
4
|
-
|
5
|
-
def acquire(names)
|
6
|
-
blocked = false
|
7
|
-
unblocked = []
|
8
|
-
|
9
|
-
queues(names).select {|queue|
|
10
|
-
next false if blocked and not unblocked.include?(queue.name)
|
11
|
-
|
12
|
-
if not queue.paused? and queue.blocking? and queue.busy > 0
|
13
|
-
blocked = true
|
14
|
-
unblocked = queue.unblocked || []
|
15
|
-
end
|
16
|
-
|
17
|
-
queue.acquire
|
18
|
-
}.map(&:name)
|
19
|
-
end
|
20
|
-
|
21
|
-
def release(names)
|
22
|
-
queues(names).each(&:release)
|
23
|
-
end
|
24
|
-
|
25
|
-
private
|
26
|
-
|
27
|
-
def queues(names)
|
28
|
-
names.map {|name| Sidekiq::Queue[name] }
|
29
|
-
end
|
30
|
-
end
|
31
|
-
end
|
@@ -1,61 +0,0 @@
|
|
1
|
-
module Sidekiq::LimitFetch::Local
|
2
|
-
class Semaphore
|
3
|
-
attr_reader :limit, :busy, :unblocked
|
4
|
-
|
5
|
-
def initialize(name)
|
6
|
-
@name = name
|
7
|
-
@lock = Mutex.new
|
8
|
-
@busy = 0
|
9
|
-
@paused = false
|
10
|
-
end
|
11
|
-
|
12
|
-
def limit=(value)
|
13
|
-
@lock.synchronize do
|
14
|
-
@limit = value
|
15
|
-
end
|
16
|
-
end
|
17
|
-
|
18
|
-
def acquire
|
19
|
-
return if @paused
|
20
|
-
@lock.synchronize do
|
21
|
-
@busy += 1 if not @limit or @limit > @busy
|
22
|
-
end
|
23
|
-
end
|
24
|
-
|
25
|
-
def release
|
26
|
-
@lock.synchronize do
|
27
|
-
@busy -= 1
|
28
|
-
end
|
29
|
-
end
|
30
|
-
|
31
|
-
def pause
|
32
|
-
@paused = true
|
33
|
-
end
|
34
|
-
|
35
|
-
def unpause
|
36
|
-
@paused = false
|
37
|
-
end
|
38
|
-
|
39
|
-
def paused?
|
40
|
-
@paused
|
41
|
-
end
|
42
|
-
|
43
|
-
def block
|
44
|
-
@block = true
|
45
|
-
end
|
46
|
-
|
47
|
-
def block_except(*queues)
|
48
|
-
raise ArgumentError if queues.empty?
|
49
|
-
@unblocked = queues
|
50
|
-
@block = true
|
51
|
-
end
|
52
|
-
|
53
|
-
def unblock
|
54
|
-
@block = false
|
55
|
-
end
|
56
|
-
|
57
|
-
def blocking?
|
58
|
-
@block
|
59
|
-
end
|
60
|
-
end
|
61
|
-
end
|