sidekiq-limit_fetch 1.6 → 1.7
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/lib/sidekiq/extensions/queue.rb +9 -12
- data/lib/sidekiq/limit_fetch.rb +1 -3
- data/lib/sidekiq/limit_fetch/global/monitor.rb +4 -2
- data/lib/sidekiq/limit_fetch/global/selector.rb +5 -5
- data/lib/sidekiq/limit_fetch/global/semaphore.rb +13 -1
- data/lib/sidekiq/limit_fetch/queues.rb +4 -6
- data/lib/sidekiq/limit_fetch/unit_of_work.rb +6 -0
- data/sidekiq-limit_fetch.gemspec +1 -1
- data/spec/sidekiq/extensions/queue_spec.rb +45 -57
- data/spec/sidekiq/limit_fetch/global/monitor_spec.rb +2 -13
- data/spec/sidekiq/limit_fetch/queues_spec.rb +44 -69
- data/spec/sidekiq/limit_fetch/semaphore_spec.rb +35 -54
- data/spec/sidekiq/limit_fetch_spec.rb +23 -23
- data/spec/spec_helper.rb +7 -7
- metadata +1 -3
- data/lib/sidekiq/limit_fetch/local/selector.rb +0 -31
- data/lib/sidekiq/limit_fetch/local/semaphore.rb +0 -61
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 4c5121de39e03363b90b64633028f65ed5cf6983
|
4
|
+
data.tar.gz: 935a28cd47181070776e71fe60db08165cab2ffe
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 09f48f0e5dce160c809b4fdad660f35c0fe2ac27ee3f6d850d094fc2e995adc70c3915cefb2ff4bc3a3aea2785c4f2a5edf3ca65c0692aea923dfbcd8ccdb2f4
|
7
|
+
data.tar.gz: a4d226be6b1df6fad9ab438dfd60e8e29235c89540f209c3399c5dd3ef24781184e653af3ae7cdbc67d08104f3a8fa16d6a44084ca36ecf3780d14e024579d78
|
@@ -3,20 +3,17 @@ module Sidekiq
|
|
3
3
|
extend LimitFetch::Singleton, Forwardable
|
4
4
|
|
5
5
|
def_delegators :lock,
|
6
|
-
:limit,
|
7
|
-
:acquire,
|
8
|
-
:pause,
|
9
|
-
:block,
|
10
|
-
:paused?,
|
11
|
-
:unblocked,
|
12
|
-
:busy
|
6
|
+
:limit, :limit=,
|
7
|
+
:acquire, :release,
|
8
|
+
:pause, :unpause,
|
9
|
+
:block, :unblock,
|
10
|
+
:paused?, :blocking?,
|
11
|
+
:unblocked, :block_except,
|
12
|
+
:probed, :busy,
|
13
|
+
:increase_busy, :decrease_busy
|
13
14
|
|
14
15
|
def lock
|
15
|
-
@lock ||=
|
16
|
-
end
|
17
|
-
|
18
|
-
def mode
|
19
|
-
Sidekiq.options[:local] ? LimitFetch::Local : LimitFetch::Global
|
16
|
+
@lock ||= LimitFetch::Global::Semaphore.new name
|
20
17
|
end
|
21
18
|
end
|
22
19
|
end
|
data/lib/sidekiq/limit_fetch.rb
CHANGED
@@ -6,8 +6,6 @@ class Sidekiq::LimitFetch
|
|
6
6
|
|
7
7
|
require_relative 'limit_fetch/singleton'
|
8
8
|
require_relative 'limit_fetch/queues'
|
9
|
-
require_relative 'limit_fetch/local/semaphore'
|
10
|
-
require_relative 'limit_fetch/local/selector'
|
11
9
|
require_relative 'limit_fetch/global/semaphore'
|
12
10
|
require_relative 'limit_fetch/global/selector'
|
13
11
|
require_relative 'limit_fetch/global/monitor'
|
@@ -20,7 +18,7 @@ class Sidekiq::LimitFetch
|
|
20
18
|
end
|
21
19
|
|
22
20
|
def initialize(options)
|
23
|
-
Global::Monitor.start!
|
21
|
+
Global::Monitor.start!
|
24
22
|
@queues = Queues.new options
|
25
23
|
end
|
26
24
|
|
@@ -37,8 +37,10 @@ module Sidekiq::LimitFetch::Global
|
|
37
37
|
next if it.get heartbeat_key processor
|
38
38
|
|
39
39
|
it.del processor_key processor
|
40
|
-
|
41
|
-
it.
|
40
|
+
%w(limit_fetch:probed:* limit_fetch:busy:*).each do |pattern|
|
41
|
+
it.keys(pattern).each do |queue|
|
42
|
+
it.lrem queue, 0, processor
|
43
|
+
end
|
42
44
|
end
|
43
45
|
end
|
44
46
|
end
|
@@ -60,7 +60,7 @@ module Sidekiq::LimitFetch::Global
|
|
60
60
|
|
61
61
|
for _, queue in ipairs(queues) do
|
62
62
|
if not blocking_mode or unblocked[queue] then
|
63
|
-
local
|
63
|
+
local probed_key = namespace..'probed:'..queue
|
64
64
|
local pause_key = namespace..'pause:'..queue
|
65
65
|
local paused = redis.call('get', pause_key)
|
66
66
|
|
@@ -72,7 +72,7 @@ module Sidekiq::LimitFetch::Global
|
|
72
72
|
local can_block = redis.call('get', block_key)
|
73
73
|
|
74
74
|
if can_block or queue_limit then
|
75
|
-
queue_locks = redis.call('llen',
|
75
|
+
queue_locks = redis.call('llen', probed_key)
|
76
76
|
end
|
77
77
|
|
78
78
|
blocking_mode = can_block and queue_locks > 0
|
@@ -84,7 +84,7 @@ module Sidekiq::LimitFetch::Global
|
|
84
84
|
end
|
85
85
|
|
86
86
|
if not queue_limit or queue_limit > queue_locks then
|
87
|
-
redis.call('rpush',
|
87
|
+
redis.call('rpush', probed_key, worker_name)
|
88
88
|
table.insert(available, queue)
|
89
89
|
end
|
90
90
|
end
|
@@ -102,8 +102,8 @@ module Sidekiq::LimitFetch::Global
|
|
102
102
|
local queues = ARGV
|
103
103
|
|
104
104
|
for _, queue in ipairs(queues) do
|
105
|
-
local
|
106
|
-
redis.call('lrem',
|
105
|
+
local probed_key = namespace..'probed:'..queue
|
106
|
+
redis.call('lrem', probed_key, 1, worker_name)
|
107
107
|
end
|
108
108
|
LUA
|
109
109
|
end
|
@@ -23,13 +23,25 @@ module Sidekiq::LimitFetch::Global
|
|
23
23
|
end
|
24
24
|
|
25
25
|
def release
|
26
|
-
Selector.
|
26
|
+
redis {|it| it.lrem "#{PREFIX}:probed:#@name", 1, Selector.uuid }
|
27
27
|
end
|
28
28
|
|
29
29
|
def busy
|
30
30
|
redis {|it| it.llen "#{PREFIX}:busy:#@name" }
|
31
31
|
end
|
32
32
|
|
33
|
+
def increase_busy
|
34
|
+
redis {|it| it.rpush "#{PREFIX}:busy:#@name", Selector.uuid }
|
35
|
+
end
|
36
|
+
|
37
|
+
def decrease_busy
|
38
|
+
redis {|it| it.lrem "#{PREFIX}:busy:#@name", 1, Selector.uuid }
|
39
|
+
end
|
40
|
+
|
41
|
+
def probed
|
42
|
+
redis {|it| it.llen "#{PREFIX}:probed:#@name" }
|
43
|
+
end
|
44
|
+
|
33
45
|
def pause
|
34
46
|
redis {|it| it.set "#{PREFIX}:pause:#@name", true }
|
35
47
|
end
|
@@ -1,19 +1,17 @@
|
|
1
1
|
class Sidekiq::LimitFetch
|
2
2
|
class Queues
|
3
3
|
THREAD_KEY = :acquired_queues
|
4
|
-
attr_reader :selector
|
5
4
|
|
6
5
|
def initialize(options)
|
7
6
|
@queues = options[:queues]
|
8
7
|
options[:strict] ? strict_order! : weighted_order!
|
9
8
|
|
10
|
-
set_selector options[:local]
|
11
9
|
set_limits options[:limits]
|
12
10
|
set_blocks options[:blocking]
|
13
11
|
end
|
14
12
|
|
15
13
|
def acquire
|
16
|
-
|
14
|
+
selector.acquire(ordered_queues)
|
17
15
|
.tap {|it| save it }
|
18
16
|
.map {|it| "queue:#{it}" }
|
19
17
|
end
|
@@ -21,13 +19,13 @@ class Sidekiq::LimitFetch
|
|
21
19
|
def release_except(full_name)
|
22
20
|
queues = restore
|
23
21
|
queues.delete full_name[/queue:(.*)/, 1] if full_name
|
24
|
-
|
22
|
+
selector.release queues
|
25
23
|
end
|
26
24
|
|
27
25
|
private
|
28
26
|
|
29
|
-
def
|
30
|
-
|
27
|
+
def selector
|
28
|
+
Global::Selector
|
31
29
|
end
|
32
30
|
|
33
31
|
def set_limits(limits)
|
data/sidekiq-limit_fetch.gemspec
CHANGED
@@ -28,73 +28,61 @@ describe Sidekiq::Queue do
|
|
28
28
|
let(:name) { 'example' }
|
29
29
|
let(:queue) { Sidekiq::Queue[name] }
|
30
30
|
|
31
|
-
|
32
|
-
|
33
|
-
|
34
|
-
end
|
35
|
-
|
36
|
-
it 'should be pausable' do
|
37
|
-
queue.pause
|
38
|
-
queue.acquire.should_not be
|
39
|
-
end
|
40
|
-
|
41
|
-
it 'should be continuable' do
|
42
|
-
queue.pause
|
43
|
-
queue.unpause
|
44
|
-
queue.acquire.should be
|
45
|
-
end
|
31
|
+
it 'should be available' do
|
32
|
+
queue.acquire.should be
|
33
|
+
end
|
46
34
|
|
47
|
-
|
48
|
-
|
49
|
-
|
50
|
-
|
51
|
-
end
|
35
|
+
it 'should be pausable' do
|
36
|
+
queue.pause
|
37
|
+
queue.acquire.should_not be
|
38
|
+
end
|
52
39
|
|
53
|
-
|
54
|
-
|
55
|
-
|
56
|
-
|
57
|
-
|
58
|
-
end
|
40
|
+
it 'should be continuable' do
|
41
|
+
queue.pause
|
42
|
+
queue.unpause
|
43
|
+
queue.acquire.should be
|
44
|
+
end
|
59
45
|
|
60
|
-
|
61
|
-
|
62
|
-
|
63
|
-
|
64
|
-
|
46
|
+
it 'should be limitable' do
|
47
|
+
queue.limit = 1
|
48
|
+
queue.acquire.should be
|
49
|
+
queue.acquire.should_not be
|
50
|
+
end
|
65
51
|
|
66
|
-
|
67
|
-
|
68
|
-
|
69
|
-
|
70
|
-
|
71
|
-
|
52
|
+
it 'should be resizable' do
|
53
|
+
queue.limit = 0
|
54
|
+
queue.acquire.should_not be
|
55
|
+
queue.limit = nil
|
56
|
+
queue.acquire.should be
|
57
|
+
end
|
72
58
|
|
73
|
-
|
74
|
-
|
75
|
-
|
76
|
-
|
77
|
-
|
78
|
-
queue.should_not be_paused
|
79
|
-
end
|
59
|
+
it 'should be countable' do
|
60
|
+
queue.limit = 3
|
61
|
+
5.times { queue.acquire }
|
62
|
+
queue.probed.should == 3
|
63
|
+
end
|
80
64
|
|
81
|
-
|
82
|
-
|
83
|
-
|
84
|
-
|
85
|
-
|
86
|
-
queue.should_not be_blocking
|
87
|
-
end
|
65
|
+
it 'should be releasable' do
|
66
|
+
queue.acquire
|
67
|
+
queue.probed.should == 1
|
68
|
+
queue.release
|
69
|
+
queue.probed.should == 0
|
88
70
|
end
|
89
71
|
|
90
|
-
|
91
|
-
|
92
|
-
|
72
|
+
it 'should tell if paused' do
|
73
|
+
queue.should_not be_paused
|
74
|
+
queue.pause
|
75
|
+
queue.should be_paused
|
76
|
+
queue.unpause
|
77
|
+
queue.should_not be_paused
|
93
78
|
end
|
94
79
|
|
95
|
-
|
96
|
-
|
97
|
-
|
80
|
+
it 'should tell if blocking' do
|
81
|
+
queue.should_not be_blocking
|
82
|
+
queue.block
|
83
|
+
queue.should be_blocking
|
84
|
+
queue.unblock
|
85
|
+
queue.should_not be_blocking
|
98
86
|
end
|
99
87
|
end
|
100
88
|
end
|
@@ -7,17 +7,6 @@ describe Sidekiq::LimitFetch::Global::Monitor do
|
|
7
7
|
let(:name) { 'default' }
|
8
8
|
|
9
9
|
before :each do
|
10
|
-
# namespaces = [
|
11
|
-
# described_class::PROCESSOR_NAMESPACE,
|
12
|
-
# described_class::HEARTBEAT_NAMESPACE
|
13
|
-
# ]
|
14
|
-
|
15
|
-
# Sidekiq.redis do |it|
|
16
|
-
# namespaces.flat_map {|namespace|
|
17
|
-
# it.keys(namespace + '*')
|
18
|
-
# }.each {|key| it.del key }
|
19
|
-
# end
|
20
|
-
|
21
10
|
monitor
|
22
11
|
end
|
23
12
|
|
@@ -31,11 +20,11 @@ describe Sidekiq::LimitFetch::Global::Monitor do
|
|
31
20
|
it 'should remove invalidated old locks' do
|
32
21
|
2.times { queue.acquire }
|
33
22
|
sleep 2*ttl
|
34
|
-
queue.
|
23
|
+
queue.probed.should == 2
|
35
24
|
|
36
25
|
described_class.stub :update_heartbeat
|
37
26
|
sleep 2*ttl
|
38
|
-
queue.
|
27
|
+
queue.probed.should == 0
|
39
28
|
end
|
40
29
|
end
|
41
30
|
end
|
@@ -6,97 +6,72 @@ describe Sidekiq::LimitFetch::Queues do
|
|
6
6
|
let(:queues) { %w[queue1 queue2] }
|
7
7
|
let(:limits) {{ 'queue1' => 3 }}
|
8
8
|
let(:strict) { true }
|
9
|
-
let(:local) {}
|
10
9
|
let(:blocking) {}
|
11
10
|
|
12
11
|
let(:options) do
|
13
12
|
{ queues: queues,
|
14
13
|
limits: limits,
|
15
14
|
strict: strict,
|
16
|
-
local: local,
|
17
15
|
blocking: blocking }
|
18
16
|
end
|
19
17
|
|
20
|
-
|
21
|
-
|
18
|
+
it 'should acquire queues' do
|
19
|
+
subject.acquire
|
20
|
+
Sidekiq::Queue['queue1'].probed.should == 1
|
21
|
+
Sidekiq::Queue['queue2'].probed.should == 1
|
22
22
|
end
|
23
23
|
|
24
|
-
|
25
|
-
|
26
|
-
|
27
|
-
|
28
|
-
Sidekiq::Queue['queue2'].busy.should == 1
|
29
|
-
end
|
30
|
-
|
31
|
-
it 'should acquire dynamically blocking queues' do
|
32
|
-
subject.acquire
|
33
|
-
Sidekiq::Queue['queue1'].busy.should == 1
|
34
|
-
Sidekiq::Queue['queue2'].busy.should == 1
|
35
|
-
|
36
|
-
Sidekiq::Queue['queue1'].block
|
37
|
-
|
38
|
-
subject.acquire
|
39
|
-
Sidekiq::Queue['queue1'].busy.should == 2
|
40
|
-
Sidekiq::Queue['queue2'].busy.should == 1
|
41
|
-
end
|
42
|
-
|
43
|
-
it 'should block except given queues' do
|
44
|
-
Sidekiq::Queue['queue1'].block_except 'queue2'
|
45
|
-
subject.acquire
|
46
|
-
Sidekiq::Queue['queue1'].busy.should == 1
|
47
|
-
Sidekiq::Queue['queue2'].busy.should == 1
|
48
|
-
|
49
|
-
Sidekiq::Queue['queue1'].block_except 'queue404'
|
50
|
-
subject.acquire
|
51
|
-
Sidekiq::Queue['queue1'].busy.should == 2
|
52
|
-
Sidekiq::Queue['queue2'].busy.should == 1
|
53
|
-
end
|
24
|
+
it 'should acquire dynamically blocking queues' do
|
25
|
+
subject.acquire
|
26
|
+
Sidekiq::Queue['queue1'].probed.should == 1
|
27
|
+
Sidekiq::Queue['queue2'].probed.should == 1
|
54
28
|
|
55
|
-
|
56
|
-
subject.acquire
|
57
|
-
subject.release_except nil
|
58
|
-
Sidekiq::Queue['queue1'].busy.should == 0
|
59
|
-
Sidekiq::Queue['queue2'].busy.should == 0
|
60
|
-
end
|
29
|
+
Sidekiq::Queue['queue1'].block
|
61
30
|
|
62
|
-
|
63
|
-
|
64
|
-
|
65
|
-
|
66
|
-
Sidekiq::Queue['queue2'].busy.should == 0
|
67
|
-
end
|
31
|
+
subject.acquire
|
32
|
+
Sidekiq::Queue['queue1'].probed.should == 2
|
33
|
+
Sidekiq::Queue['queue2'].probed.should == 1
|
34
|
+
end
|
68
35
|
|
69
|
-
|
70
|
-
|
71
|
-
|
72
|
-
|
73
|
-
|
36
|
+
it 'should block except given queues' do
|
37
|
+
Sidekiq::Queue['queue1'].block_except 'queue2'
|
38
|
+
subject.acquire
|
39
|
+
Sidekiq::Queue['queue1'].probed.should == 1
|
40
|
+
Sidekiq::Queue['queue2'].probed.should == 1
|
74
41
|
|
75
|
-
|
76
|
-
|
42
|
+
Sidekiq::Queue['queue1'].block_except 'queue404'
|
43
|
+
subject.acquire
|
44
|
+
Sidekiq::Queue['queue1'].probed.should == 2
|
45
|
+
Sidekiq::Queue['queue2'].probed.should == 1
|
46
|
+
end
|
77
47
|
|
78
|
-
|
79
|
-
|
80
|
-
|
81
|
-
|
82
|
-
|
83
|
-
end
|
48
|
+
it 'should release queues' do
|
49
|
+
subject.acquire
|
50
|
+
subject.release_except nil
|
51
|
+
Sidekiq::Queue['queue1'].probed.should == 0
|
52
|
+
Sidekiq::Queue['queue2'].probed.should == 0
|
84
53
|
end
|
85
54
|
|
86
|
-
|
87
|
-
|
55
|
+
it 'should release queues except selected' do
|
56
|
+
subject.acquire
|
57
|
+
subject.release_except 'queue:queue1'
|
58
|
+
Sidekiq::Queue['queue1'].probed.should == 1
|
59
|
+
Sidekiq::Queue['queue2'].probed.should == 0
|
60
|
+
end
|
88
61
|
|
89
|
-
|
90
|
-
|
91
|
-
|
62
|
+
it 'should release when no queues was acquired' do
|
63
|
+
queues.each {|name| Sidekiq::Queue[name].pause }
|
64
|
+
subject.acquire
|
65
|
+
-> { subject.release_except nil }.should_not raise_exception
|
92
66
|
end
|
93
67
|
|
94
|
-
context '
|
95
|
-
let(:
|
96
|
-
it_should_behave_like :selector
|
68
|
+
context 'blocking' do
|
69
|
+
let(:blocking) { %w(queue1) }
|
97
70
|
|
98
|
-
it 'should
|
99
|
-
subject.
|
71
|
+
it 'should acquire blocking queues' do
|
72
|
+
3.times { subject.acquire }
|
73
|
+
Sidekiq::Queue['queue1'].probed.should == 3
|
74
|
+
Sidekiq::Queue['queue2'].probed.should == 1
|
100
75
|
end
|
101
76
|
end
|
102
77
|
|
@@ -1,68 +1,49 @@
|
|
1
1
|
require 'spec_helper'
|
2
2
|
|
3
3
|
describe 'semaphore' do
|
4
|
-
|
5
|
-
|
6
|
-
subject.limit.should_not be
|
7
|
-
end
|
8
|
-
|
9
|
-
it 'should set limit' do
|
10
|
-
subject.limit = 4
|
11
|
-
subject.limit.should == 4
|
12
|
-
end
|
13
|
-
|
14
|
-
it 'should acquire and count active tasks' do
|
15
|
-
3.times { subject.acquire }
|
16
|
-
subject.busy.should == 3
|
17
|
-
end
|
18
|
-
|
19
|
-
it 'should acquire tasks with regard to limit' do
|
20
|
-
subject.limit = 4
|
21
|
-
6.times { subject.acquire }
|
22
|
-
subject.busy.should == 4
|
23
|
-
end
|
4
|
+
let(:name) { 'default' }
|
5
|
+
subject { Sidekiq::LimitFetch::Global::Semaphore.new name }
|
24
6
|
|
25
|
-
|
26
|
-
|
27
|
-
|
28
|
-
subject.busy.should == 3
|
29
|
-
end
|
7
|
+
it 'should have no limit by default' do
|
8
|
+
subject.limit.should_not be
|
9
|
+
end
|
30
10
|
|
31
|
-
|
32
|
-
|
33
|
-
|
34
|
-
|
35
|
-
subject.busy.should == 3
|
36
|
-
2.times { subject.release }
|
37
|
-
subject.busy.should == 1
|
38
|
-
end
|
11
|
+
it 'should set limit' do
|
12
|
+
subject.limit = 4
|
13
|
+
subject.limit.should == 4
|
14
|
+
end
|
39
15
|
|
40
|
-
|
41
|
-
|
42
|
-
|
43
|
-
subject.unpause
|
44
|
-
2.times { subject.acquire }
|
45
|
-
subject.busy.should == 2
|
46
|
-
end
|
16
|
+
it 'should acquire and count active tasks' do
|
17
|
+
3.times { subject.acquire }
|
18
|
+
subject.probed.should == 3
|
47
19
|
end
|
48
20
|
|
49
|
-
|
21
|
+
it 'should acquire tasks with regard to limit' do
|
22
|
+
subject.limit = 4
|
23
|
+
6.times { subject.acquire }
|
24
|
+
subject.probed.should == 4
|
25
|
+
end
|
50
26
|
|
51
|
-
|
52
|
-
|
53
|
-
|
27
|
+
it 'should release active tasks' do
|
28
|
+
6.times { subject.acquire }
|
29
|
+
3.times { subject.release }
|
30
|
+
subject.probed.should == 3
|
54
31
|
end
|
55
32
|
|
56
|
-
|
57
|
-
|
58
|
-
|
33
|
+
it 'should pause tasks' do
|
34
|
+
3.times { subject.acquire }
|
35
|
+
subject.pause
|
36
|
+
2.times { subject.acquire }
|
37
|
+
subject.probed.should == 3
|
38
|
+
2.times { subject.release }
|
39
|
+
subject.probed.should == 1
|
40
|
+
end
|
59
41
|
|
60
|
-
|
61
|
-
|
62
|
-
|
63
|
-
|
64
|
-
|
65
|
-
|
66
|
-
end
|
42
|
+
it 'should unpause tasks' do
|
43
|
+
subject.pause
|
44
|
+
3.times { subject.acquire }
|
45
|
+
subject.unpause
|
46
|
+
2.times { subject.acquire }
|
47
|
+
subject.probed.should == 2
|
67
48
|
end
|
68
49
|
end
|
@@ -11,37 +11,37 @@ describe Sidekiq::LimitFetch do
|
|
11
11
|
end
|
12
12
|
|
13
13
|
subject { described_class.new options }
|
14
|
-
let(:options) {{ queues: queues, limits: limits
|
14
|
+
let(:options) {{ queues: queues, limits: limits }}
|
15
15
|
let(:queues) { %w(queue1 queue1 queue2 queue2) }
|
16
16
|
let(:limits) {{ 'queue1' => 1, 'queue2' => 2 }}
|
17
17
|
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
work.message.should == 'task1'
|
18
|
+
it 'should acquire lock on queue for execution' do
|
19
|
+
work = subject.retrieve_work
|
20
|
+
work.queue_name.should == 'queue1'
|
21
|
+
work.message.should == 'task1'
|
23
22
|
|
24
|
-
|
25
|
-
|
23
|
+
Sidekiq::Queue['queue1'].busy.should == 1
|
24
|
+
Sidekiq::Queue['queue2'].busy.should == 0
|
26
25
|
|
27
|
-
|
28
|
-
|
26
|
+
subject.retrieve_work.should_not be
|
27
|
+
work.requeue
|
29
28
|
|
30
|
-
|
31
|
-
|
29
|
+
Sidekiq::Queue['queue1'].busy.should == 0
|
30
|
+
Sidekiq::Queue['queue2'].busy.should == 0
|
32
31
|
|
33
|
-
|
34
|
-
|
35
|
-
end
|
36
|
-
end
|
32
|
+
work = subject.retrieve_work
|
33
|
+
work.message.should == 'task2'
|
37
34
|
|
38
|
-
|
39
|
-
|
40
|
-
|
41
|
-
|
35
|
+
Sidekiq::Queue['queue1'].busy.should == 1
|
36
|
+
Sidekiq::Queue['queue2'].busy.should == 0
|
37
|
+
|
38
|
+
subject.retrieve_work.should_not be
|
39
|
+
work.acknowledge
|
40
|
+
|
41
|
+
Sidekiq::Queue['queue1'].busy.should == 0
|
42
|
+
Sidekiq::Queue['queue2'].busy.should == 0
|
42
43
|
|
43
|
-
|
44
|
-
|
45
|
-
it_behaves_like :strategy
|
44
|
+
work = subject.retrieve_work
|
45
|
+
work.message.should == 'task1'
|
46
46
|
end
|
47
47
|
end
|
data/spec/spec_helper.rb
CHANGED
@@ -4,15 +4,15 @@ require 'sidekiq/fetch'
|
|
4
4
|
|
5
5
|
RSpec.configure do |config|
|
6
6
|
config.before :each do
|
7
|
-
Sidekiq::Queue.instance_variable_set :@instances, {}
|
8
|
-
Sidekiq.options[:local] = defined?(local) ? local : nil
|
9
|
-
|
10
7
|
Sidekiq.redis do |it|
|
11
8
|
clean_redis = ->(queue) do
|
12
|
-
it.
|
13
|
-
|
14
|
-
|
15
|
-
|
9
|
+
it.pipelined do
|
10
|
+
it.del "limit_fetch:limit:#{queue}"
|
11
|
+
it.del "limit_fetch:busy:#{queue}"
|
12
|
+
it.del "limit_fetch:probed:#{queue}"
|
13
|
+
it.del "limit_fetch:pause:#{queue}"
|
14
|
+
it.del "limit_fetch:block:#{queue}"
|
15
|
+
end
|
16
16
|
end
|
17
17
|
|
18
18
|
clean_redis.call(name) if defined?(name)
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: sidekiq-limit_fetch
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: '1.
|
4
|
+
version: '1.7'
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- brainopia
|
@@ -77,8 +77,6 @@ files:
|
|
77
77
|
- lib/sidekiq/limit_fetch/global/monitor.rb
|
78
78
|
- lib/sidekiq/limit_fetch/global/selector.rb
|
79
79
|
- lib/sidekiq/limit_fetch/global/semaphore.rb
|
80
|
-
- lib/sidekiq/limit_fetch/local/selector.rb
|
81
|
-
- lib/sidekiq/limit_fetch/local/semaphore.rb
|
82
80
|
- lib/sidekiq/limit_fetch/queues.rb
|
83
81
|
- lib/sidekiq/limit_fetch/singleton.rb
|
84
82
|
- lib/sidekiq/limit_fetch/unit_of_work.rb
|
@@ -1,31 +0,0 @@
|
|
1
|
-
module Sidekiq::LimitFetch::Local
|
2
|
-
module Selector
|
3
|
-
extend self
|
4
|
-
|
5
|
-
def acquire(names)
|
6
|
-
blocked = false
|
7
|
-
unblocked = []
|
8
|
-
|
9
|
-
queues(names).select {|queue|
|
10
|
-
next false if blocked and not unblocked.include?(queue.name)
|
11
|
-
|
12
|
-
if not queue.paused? and queue.blocking? and queue.busy > 0
|
13
|
-
blocked = true
|
14
|
-
unblocked = queue.unblocked || []
|
15
|
-
end
|
16
|
-
|
17
|
-
queue.acquire
|
18
|
-
}.map(&:name)
|
19
|
-
end
|
20
|
-
|
21
|
-
def release(names)
|
22
|
-
queues(names).each(&:release)
|
23
|
-
end
|
24
|
-
|
25
|
-
private
|
26
|
-
|
27
|
-
def queues(names)
|
28
|
-
names.map {|name| Sidekiq::Queue[name] }
|
29
|
-
end
|
30
|
-
end
|
31
|
-
end
|
@@ -1,61 +0,0 @@
|
|
1
|
-
module Sidekiq::LimitFetch::Local
|
2
|
-
class Semaphore
|
3
|
-
attr_reader :limit, :busy, :unblocked
|
4
|
-
|
5
|
-
def initialize(name)
|
6
|
-
@name = name
|
7
|
-
@lock = Mutex.new
|
8
|
-
@busy = 0
|
9
|
-
@paused = false
|
10
|
-
end
|
11
|
-
|
12
|
-
def limit=(value)
|
13
|
-
@lock.synchronize do
|
14
|
-
@limit = value
|
15
|
-
end
|
16
|
-
end
|
17
|
-
|
18
|
-
def acquire
|
19
|
-
return if @paused
|
20
|
-
@lock.synchronize do
|
21
|
-
@busy += 1 if not @limit or @limit > @busy
|
22
|
-
end
|
23
|
-
end
|
24
|
-
|
25
|
-
def release
|
26
|
-
@lock.synchronize do
|
27
|
-
@busy -= 1
|
28
|
-
end
|
29
|
-
end
|
30
|
-
|
31
|
-
def pause
|
32
|
-
@paused = true
|
33
|
-
end
|
34
|
-
|
35
|
-
def unpause
|
36
|
-
@paused = false
|
37
|
-
end
|
38
|
-
|
39
|
-
def paused?
|
40
|
-
@paused
|
41
|
-
end
|
42
|
-
|
43
|
-
def block
|
44
|
-
@block = true
|
45
|
-
end
|
46
|
-
|
47
|
-
def block_except(*queues)
|
48
|
-
raise ArgumentError if queues.empty?
|
49
|
-
@unblocked = queues
|
50
|
-
@block = true
|
51
|
-
end
|
52
|
-
|
53
|
-
def unblock
|
54
|
-
@block = false
|
55
|
-
end
|
56
|
-
|
57
|
-
def blocking?
|
58
|
-
@block
|
59
|
-
end
|
60
|
-
end
|
61
|
-
end
|