sqskiq 0.0.3 → 0.0.4
Sign up to get free protection for your applications and to get access to all the features.
- data/lib/sqskiq.rb +30 -29
- data/lib/sqskiq/batch_process.rb +2 -2
- data/lib/sqskiq/manager.rb +17 -7
- data/spec/manager_spec.rb +78 -33
- data/spec/sqskiq_spec.rb +20 -20
- metadata +1 -1
data/lib/sqskiq.rb
CHANGED
@@ -7,58 +7,59 @@ require 'sqskiq/batch_process'
|
|
7
7
|
|
8
8
|
module Sqskiq
|
9
9
|
|
10
|
-
|
11
|
-
|
10
|
+
##
|
11
|
+
# Configures and starts actor system
|
12
|
+
def self.bootstrap(worker_config, worker_class)
|
13
|
+
config = valid_config_from(worker_config)
|
14
|
+
credentials = [ @aws_access_key_id, @aws_secret_access_key, config[:queue_name] ]
|
12
15
|
|
13
|
-
|
16
|
+
Celluloid::Actor[:manager] = @manager = Manager.new(config[:empty_queue_throttle])
|
17
|
+
Celluloid::Actor[:fetcher] = @fetcher = Fetcher.pool(:size => config[:num_fetchers], :args => credentials)
|
18
|
+
Celluloid::Actor[:deleter] = @deleter = Deleter.pool(:size => config[:num_deleters], :args => credentials)
|
19
|
+
Celluloid::Actor[:processor] = @processor = Processor.pool(:size => config[:num_workers], :args => worker_class)
|
20
|
+
Celluloid::Actor[:batcher] = @batcher = BatchProcessor.pool(:size => config[:num_batches])
|
14
21
|
|
15
|
-
Celluloid::Actor[:manager] = @manager = Manager.new
|
16
|
-
Celluloid::Actor[:fetcher] = @fetcher = Fetcher.pool(:size => configured_pool_sizes[:num_fetchers], :args => params)
|
17
|
-
Celluloid::Actor[:processor] = @processor = Processor.pool(:size => configured_pool_sizes[:num_workers], :args => worker_class)
|
18
|
-
Celluloid::Actor[:batch_processor] = @batch_processor = BatchProcessor.pool(:size => configured_pool_sizes[:num_batches])
|
19
|
-
Celluloid::Actor[:deleter] = @deleter = Deleter.pool(:size => configured_pool_sizes[:num_deleters], :args => params)
|
20
|
-
|
21
22
|
configure_signal_listeners
|
22
|
-
|
23
|
+
|
23
24
|
@manager.bootstrap
|
24
25
|
while @manager.running? do
|
25
26
|
sleep 2
|
26
27
|
end
|
27
|
-
|
28
|
-
@fetcher.__shutdown__
|
29
|
-
@batch_processor.__shutdown__
|
30
|
-
@processor.__shutdown__
|
31
|
-
@deleter.__shutdown__
|
32
|
-
|
33
28
|
@manager.terminate
|
34
29
|
end
|
35
30
|
|
31
|
+
# Subscribes actors to receive system signals
|
32
|
+
# Each actor when receives a signal should execute
|
33
|
+
# appropriate code to exit cleanly
|
36
34
|
def self.configure_signal_listeners
|
37
35
|
['SIGTERM', 'TERM', 'SIGINT'].each do |signal|
|
38
36
|
trap(signal) do
|
39
37
|
@manager.publish('SIGTERM')
|
40
|
-
@
|
38
|
+
@batcher.publish('SIGTERM')
|
41
39
|
@processor.publish('SIGTERM')
|
42
40
|
end
|
43
41
|
end
|
44
42
|
end
|
45
43
|
|
46
|
-
|
47
|
-
|
48
|
-
|
49
|
-
|
50
|
-
|
51
|
-
#
|
52
|
-
# workers handling messages
|
53
|
-
# TODO: acctualy the min number must be greater than 2 because we are using
|
54
|
-
# celluloid pool, but that will be changed!
|
44
|
+
##
|
45
|
+
# checks the provided configuration
|
46
|
+
# and add the defaults when not specified
|
47
|
+
def self.valid_config_from(worker_config)
|
48
|
+
num_workers = (worker_config[:processors].nil? || worker_config[:processors].to_i < 2)? 20 : worker_config[:processors]
|
49
|
+
# messy code due to celluloid pool constraint of 2 as min pool size: see spec for better understanding
|
55
50
|
num_fetchers = num_workers / 10
|
56
51
|
num_fetchers = num_fetchers + 1 if num_workers % 10 > 0
|
57
52
|
num_fetchers = 2 if num_fetchers < 2
|
58
|
-
|
59
53
|
num_deleters = num_batches = num_fetchers
|
60
|
-
|
61
|
-
{
|
54
|
+
|
55
|
+
{
|
56
|
+
num_workers: num_workers,
|
57
|
+
num_fetchers: num_fetchers,
|
58
|
+
num_batches: num_batches,
|
59
|
+
num_deleters: num_deleters,
|
60
|
+
queue_name: worker_config[:queue_name],
|
61
|
+
empty_queue_throttle: worker_config[:empty_queue_throttle] || 0
|
62
|
+
}
|
62
63
|
end
|
63
64
|
|
64
65
|
def self.configure
|
data/lib/sqskiq/batch_process.rb
CHANGED
@@ -13,7 +13,7 @@ module Sqskiq
|
|
13
13
|
subscribe_for_shutdown
|
14
14
|
end
|
15
15
|
|
16
|
-
def
|
16
|
+
def process(messages)
|
17
17
|
process_result = []
|
18
18
|
messages.each do |message|
|
19
19
|
process_result << @processor.future.process(message)
|
@@ -30,7 +30,7 @@ module Sqskiq
|
|
30
30
|
end
|
31
31
|
end
|
32
32
|
|
33
|
-
@manager.async.
|
33
|
+
@manager.async.batch_done(success_messages)
|
34
34
|
end
|
35
35
|
end
|
36
36
|
|
data/lib/sqskiq/manager.rb
CHANGED
@@ -7,33 +7,43 @@ module Sqskiq
|
|
7
7
|
include Celluloid
|
8
8
|
include Sqskiq::SignalHandler
|
9
9
|
|
10
|
-
|
11
|
-
|
10
|
+
@empty_queue = false
|
11
|
+
|
12
|
+
def initialize(empty_queue_throttle)
|
13
|
+
@empty_queue_throttle = empty_queue_throttle
|
14
|
+
subscribe_for_shutdown
|
12
15
|
end
|
13
16
|
|
14
17
|
def bootstrap
|
15
18
|
@fetcher = Celluloid::Actor[:fetcher]
|
16
|
-
@
|
19
|
+
@batcher = Celluloid::Actor[:batcher]
|
17
20
|
@deleter = Celluloid::Actor[:deleter]
|
18
21
|
|
19
22
|
new_fetch(@fetcher.size)
|
20
23
|
end
|
21
24
|
|
22
25
|
def fetch_done(messages)
|
23
|
-
@
|
26
|
+
@empty_queue = messages.empty?
|
27
|
+
@batcher.async.process(messages) unless @shutting_down
|
24
28
|
end
|
25
29
|
|
26
|
-
def
|
30
|
+
def batch_done(messages)
|
27
31
|
@deleter.async.delete(messages)
|
28
32
|
new_fetch(1)
|
29
33
|
end
|
30
34
|
|
31
35
|
def new_fetch(num)
|
32
|
-
|
36
|
+
after(throttle) do
|
37
|
+
num.times { @fetcher.async.fetch unless @shutting_down }
|
38
|
+
end
|
33
39
|
end
|
34
40
|
|
35
41
|
def running?
|
36
|
-
not (@shutting_down and @deleter.busy_size == 0 and @
|
42
|
+
not (@shutting_down and @deleter.busy_size == 0 and @batcher.busy_size == 0)
|
43
|
+
end
|
44
|
+
|
45
|
+
def throttle
|
46
|
+
@empty_queue ? @empty_queue_throttle : 0
|
37
47
|
end
|
38
48
|
|
39
49
|
end
|
data/spec/manager_spec.rb
CHANGED
@@ -1,70 +1,115 @@
|
|
1
1
|
require 'spec_helper'
|
2
2
|
|
3
3
|
describe Sqskiq::Manager do
|
4
|
+
subject { Sqskiq::Manager.new(Random.rand(1..100)) }
|
5
|
+
|
4
6
|
let(:deleter) { Object.new }
|
5
7
|
let(:fetcher) { Object.new }
|
6
|
-
let(:
|
8
|
+
let(:batcher) { Object.new }
|
7
9
|
let(:shutting_down) { false }
|
8
|
-
subject = described_class.new
|
9
10
|
|
10
11
|
before do
|
11
12
|
subject.instance_variable_set(:@fetcher, fetcher)
|
12
13
|
subject.instance_variable_set(:@deleter, deleter)
|
13
|
-
subject.instance_variable_set(:@
|
14
|
+
subject.instance_variable_set(:@batcher, batcher)
|
14
15
|
subject.instance_variable_set(:@shutting_down, shutting_down)
|
15
16
|
end
|
16
17
|
|
17
18
|
describe '#running?' do
|
18
|
-
|
19
19
|
context 'when the actor system is shutting down' do
|
20
|
-
|
20
|
+
let(:shutting_down) { true }
|
21
21
|
|
22
|
-
|
23
|
-
|
24
|
-
|
25
|
-
|
26
|
-
|
27
|
-
end
|
28
|
-
end
|
22
|
+
describe 'if deleter is not empty' do
|
23
|
+
before { deleter.should_receive(:busy_size).and_return(1) }
|
24
|
+
|
25
|
+
it { should be_running }
|
26
|
+
end
|
29
27
|
|
30
|
-
|
31
|
-
|
32
|
-
|
33
|
-
|
34
|
-
|
28
|
+
describe 'if batcher is not empty' do
|
29
|
+
before do
|
30
|
+
deleter.should_receive(:busy_size).and_return(0)
|
31
|
+
batcher.should_receive(:busy_size).and_return(1)
|
32
|
+
end
|
35
33
|
|
36
|
-
|
37
|
-
|
38
|
-
end
|
39
|
-
end
|
34
|
+
it { should be_running }
|
35
|
+
end
|
40
36
|
|
41
|
-
|
37
|
+
describe 'if batcher and deleter are empties' do
|
42
38
|
before do
|
43
39
|
deleter.should_receive(:busy_size).and_return(0)
|
44
|
-
|
40
|
+
batcher.should_receive(:busy_size).and_return(0)
|
45
41
|
end
|
46
42
|
|
47
|
-
it
|
48
|
-
|
49
|
-
end
|
50
|
-
end
|
51
|
-
|
43
|
+
it { should_not be_running }
|
44
|
+
end
|
52
45
|
end
|
53
46
|
|
54
47
|
context 'when the actor system is not shutting down' do
|
55
48
|
|
56
|
-
describe 'even if
|
49
|
+
describe 'even if batcher and deleter are empties' do
|
57
50
|
before do
|
58
51
|
deleter.stub(:busy_size).and_return(0)
|
59
|
-
|
52
|
+
batcher.stub(:busy_size).and_return(0)
|
60
53
|
end
|
61
54
|
|
62
|
-
it
|
63
|
-
subject.running?.should be_true
|
64
|
-
end
|
55
|
+
it { should be_running }
|
65
56
|
end
|
66
57
|
|
67
58
|
end
|
68
59
|
end
|
60
|
+
|
61
|
+
describe '#new_fetch' do
|
62
|
+
before do
|
63
|
+
subject.instance_variable_set(:@empty_queue, empty)
|
64
|
+
subject.instance_variable_set(:@empty_queue_throttle, 10)
|
65
|
+
end
|
66
|
+
|
67
|
+
context 'if queue is not empty' do
|
68
|
+
let(:empty) { false }
|
69
|
+
|
70
|
+
it 'applies a throttle of 0 seconds' do
|
71
|
+
Sqskiq::Manager.any_instance.should_receive(:after).with(0)
|
72
|
+
subject.new_fetch(1)
|
73
|
+
end
|
74
|
+
end
|
75
|
+
|
76
|
+
context 'if queue is empty' do
|
77
|
+
let(:empty) { true }
|
78
|
+
|
79
|
+
it 'applies a throttle of empty_queue_throttle seconds' do
|
80
|
+
Sqskiq::Manager.any_instance.should_receive(:after).with(10)
|
81
|
+
subject.new_fetch(1)
|
82
|
+
end
|
83
|
+
end
|
84
|
+
end
|
85
|
+
|
86
|
+
describe '#fetch_done' do
|
87
|
+
|
88
|
+
before { subject.instance_variable_set(:@empty_queue, empty) }
|
89
|
+
|
90
|
+
context 'when at least one message has been received' do
|
91
|
+
let(:messages) { ['someMessage'] }
|
92
|
+
let(:empty) { true }
|
93
|
+
|
94
|
+
it 'sets @empty_queue to false and process the messages' do
|
95
|
+
batcher.should_receive(:async).and_return(batcher)
|
96
|
+
batcher.should_receive(:process).with(messages)
|
97
|
+
subject.fetch_done(messages)
|
98
|
+
subject.instance_variable_get(:@empty_queue).should be_false
|
99
|
+
end
|
100
|
+
end
|
101
|
+
|
102
|
+
context 'when no messages are received' do
|
103
|
+
let(:messages) { [] }
|
104
|
+
let(:empty) { false }
|
69
105
|
|
106
|
+
it 'sets @empty_queue to true and does not process the messages' do
|
107
|
+
batcher.should_receive(:async).and_return(batcher)
|
108
|
+
batcher.should_receive(:process).with(messages)
|
109
|
+
subject.fetch_done(messages)
|
110
|
+
subject.instance_variable_get(:@empty_queue).should be_true
|
111
|
+
end
|
112
|
+
end
|
113
|
+
end
|
114
|
+
|
70
115
|
end
|
data/spec/sqskiq_spec.rb
CHANGED
@@ -6,11 +6,11 @@ describe Sqskiq do
|
|
6
6
|
let(:options) { [ { processors: 1 }, ].sample }
|
7
7
|
|
8
8
|
it 'uses the defaut value of 20' do
|
9
|
-
|
10
|
-
|
11
|
-
|
12
|
-
|
13
|
-
|
9
|
+
config = Sqskiq.valid_config_from(options)
|
10
|
+
config[:num_workers].should eq(20)
|
11
|
+
config[:num_fetchers].should eq(2)
|
12
|
+
config[:num_batches].should eq(2)
|
13
|
+
config[:num_deleters].should eq(2)
|
14
14
|
end
|
15
15
|
end
|
16
16
|
|
@@ -20,11 +20,11 @@ describe Sqskiq do
|
|
20
20
|
let(:options) { { processors: [ 20, 30, 40 ].sample } }
|
21
21
|
|
22
22
|
it 'uses the the given value' do
|
23
|
-
|
24
|
-
|
25
|
-
|
26
|
-
|
27
|
-
|
23
|
+
config = Sqskiq.valid_config_from(options)
|
24
|
+
config[:num_workers].should eq(options[:processors])
|
25
|
+
config[:num_fetchers].should eq(options[:processors] / 10)
|
26
|
+
config[:num_batches].should eq(options[:processors] / 10)
|
27
|
+
config[:num_deleters].should eq(options[:processors] / 10)
|
28
28
|
end
|
29
29
|
|
30
30
|
end
|
@@ -33,11 +33,11 @@ describe Sqskiq do
|
|
33
33
|
let(:options) { { processors: [ 21, 31, 41 ].sample } }
|
34
34
|
|
35
35
|
it 'uses the the given value for the processors and apply (processors / 10) + 1 for other pool sizes' do
|
36
|
-
|
37
|
-
|
38
|
-
|
39
|
-
|
40
|
-
|
36
|
+
config = Sqskiq.valid_config_from(options)
|
37
|
+
config[:num_workers].should eq(options[:processors])
|
38
|
+
config[:num_fetchers].should eq((options[:processors] / 10) + 1)
|
39
|
+
config[:num_batches].should eq((options[:processors] / 10) + 1)
|
40
|
+
config[:num_deleters].should eq((options[:processors] / 10) + 1)
|
41
41
|
end
|
42
42
|
|
43
43
|
end
|
@@ -46,11 +46,11 @@ describe Sqskiq do
|
|
46
46
|
let(:options) { { processors: Random.rand(2..10) } }
|
47
47
|
|
48
48
|
it 'uses the the given value for the processors and apply (processors / 10) for other pool sizes' do
|
49
|
-
|
50
|
-
|
51
|
-
|
52
|
-
|
53
|
-
|
49
|
+
config = Sqskiq.valid_config_from(options)
|
50
|
+
config[:num_workers].should eq(options[:processors])
|
51
|
+
config[:num_fetchers].should eq(2)
|
52
|
+
config[:num_batches].should eq(2)
|
53
|
+
config[:num_deleters].should eq(2)
|
54
54
|
end
|
55
55
|
|
56
56
|
end
|