sqskiq 0.0.1 → 0.0.2
Sign up to get free protection for your applications and to get access to all the features.
- data/lib/sqskiq/batch_process.rb +11 -20
- data/lib/sqskiq/delete.rb +0 -1
- data/lib/sqskiq/fetch.rb +0 -13
- data/lib/sqskiq/manager.rb +8 -18
- data/lib/sqskiq/process.rb +7 -15
- data/lib/sqskiq/worker.rb +2 -0
- data/lib/sqskiq.rb +37 -20
- data/spec/manager_spec.rb +70 -0
- data/spec/processor_spec.rb +14 -0
- data/spec/spec_helper.rb +1 -0
- data/spec/sqskiq_spec.rb +60 -0
- metadata +42 -2
data/lib/sqskiq/batch_process.rb
CHANGED
@@ -1,21 +1,19 @@
|
|
1
1
|
require 'celluloid'
|
2
2
|
require 'celluloid/autostart'
|
3
|
+
require 'sqskiq/signal_handler'
|
3
4
|
|
4
5
|
module Sqskiq
|
5
6
|
class BatchProcessor
|
6
7
|
include Celluloid
|
7
|
-
include
|
8
|
-
|
8
|
+
include Sqskiq::SignalHandler
|
9
|
+
|
9
10
|
def initialize
|
10
11
|
@manager = Celluloid::Actor[:manager]
|
11
12
|
@processor = Celluloid::Actor[:processor]
|
12
|
-
|
13
|
-
subscribe_interrupt
|
13
|
+
subscribe_for_shutdown
|
14
14
|
end
|
15
15
|
|
16
16
|
def batch_process(messages)
|
17
|
-
p "processing #{messages.size} messages"
|
18
|
-
|
19
17
|
process_result = []
|
20
18
|
messages.each do |message|
|
21
19
|
process_result << @processor.future.process(message)
|
@@ -23,24 +21,17 @@ module Sqskiq
|
|
23
21
|
|
24
22
|
success_messages = []
|
25
23
|
process_result.each do |result|
|
26
|
-
|
27
|
-
|
28
|
-
|
24
|
+
|
25
|
+
unless @shutting_down
|
26
|
+
value = result.value
|
27
|
+
if value[:success]
|
28
|
+
success_messages << value[:message]
|
29
|
+
end
|
29
30
|
end
|
30
31
|
end
|
31
32
|
|
32
33
|
@manager.async.batch_process_done(success_messages)
|
33
34
|
end
|
34
|
-
|
35
|
-
def subscribe_interrupt
|
36
|
-
subscribe('SIGINT', :interrupt)
|
37
|
-
subscribe('TERM', :interrupt)
|
38
|
-
subscribe('SIGTERM', :interrupt)
|
39
|
-
end
|
40
|
-
|
41
|
-
def interrupt(signal)
|
42
|
-
self.terminate
|
43
|
-
end
|
44
|
-
|
45
35
|
end
|
36
|
+
|
46
37
|
end
|
data/lib/sqskiq/delete.rb
CHANGED
data/lib/sqskiq/fetch.rb
CHANGED
@@ -5,13 +5,10 @@ require 'sqskiq/aws'
|
|
5
5
|
module Sqskiq
|
6
6
|
class Fetcher
|
7
7
|
include Celluloid
|
8
|
-
include Celluloid::Notifications
|
9
8
|
include Sqskiq::AWS
|
10
9
|
|
11
10
|
def initialize(aws_access_key_id, aws_secret_access_key, queue_name)
|
12
11
|
init_queue(aws_access_key_id, aws_secret_access_key, queue_name)
|
13
|
-
subscribe_interrupt
|
14
|
-
|
15
12
|
@manager = Celluloid::Actor[:manager]
|
16
13
|
end
|
17
14
|
|
@@ -20,15 +17,5 @@ module Sqskiq
|
|
20
17
|
@manager.async.fetch_done(messages)
|
21
18
|
end
|
22
19
|
|
23
|
-
def subscribe_interrupt
|
24
|
-
subscribe('SIGINT', :interrupt)
|
25
|
-
subscribe('SIGTERM', :interrupt)
|
26
|
-
subscribe('TERM', :interrupt)
|
27
|
-
end
|
28
|
-
|
29
|
-
def interrupt(signal)
|
30
|
-
self.terminate
|
31
|
-
end
|
32
|
-
|
33
20
|
end
|
34
21
|
end
|
data/lib/sqskiq/manager.rb
CHANGED
@@ -1,23 +1,22 @@
|
|
1
1
|
require 'celluloid'
|
2
2
|
require 'celluloid/autostart'
|
3
|
-
|
3
|
+
require 'sqskiq/signal_handler'
|
4
4
|
|
5
5
|
module Sqskiq
|
6
6
|
class Manager
|
7
7
|
include Celluloid
|
8
|
-
include
|
9
|
-
|
8
|
+
include Sqskiq::SignalHandler
|
9
|
+
|
10
10
|
def initialize
|
11
|
-
|
12
|
-
subscribe_shutting_down
|
11
|
+
subscribe_for_shutdown
|
13
12
|
end
|
14
13
|
|
15
14
|
def bootstrap
|
16
15
|
@fetcher = Celluloid::Actor[:fetcher]
|
17
16
|
@batch_processor = Celluloid::Actor[:batch_processor]
|
18
17
|
@deleter = Celluloid::Actor[:deleter]
|
19
|
-
|
20
|
-
new_fetch(
|
18
|
+
|
19
|
+
new_fetch(@fetcher.size)
|
21
20
|
end
|
22
21
|
|
23
22
|
def fetch_done(messages)
|
@@ -25,7 +24,7 @@ module Sqskiq
|
|
25
24
|
end
|
26
25
|
|
27
26
|
def batch_process_done(messages)
|
28
|
-
@deleter.async.delete(messages)
|
27
|
+
@deleter.async.delete(messages)
|
29
28
|
new_fetch(1)
|
30
29
|
end
|
31
30
|
|
@@ -33,18 +32,9 @@ module Sqskiq
|
|
33
32
|
num.times { @fetcher.async.fetch unless @shutting_down }
|
34
33
|
end
|
35
34
|
|
36
|
-
def shutting_down(signal)
|
37
|
-
@shutting_down = true
|
38
|
-
end
|
39
|
-
|
40
35
|
def running?
|
41
|
-
not (@deleter.busy_size == 0 and @
|
36
|
+
not (@shutting_down and @deleter.busy_size == 0 and @batch_processor.busy_size == 0)
|
42
37
|
end
|
43
38
|
|
44
|
-
def subscribe_shutting_down
|
45
|
-
subscribe('SIGINT', :shutting_down)
|
46
|
-
subscribe('TERM', :shutting_down)
|
47
|
-
subscribe('SIGTERM', :shutting_down)
|
48
|
-
end
|
49
39
|
end
|
50
40
|
end
|
data/lib/sqskiq/process.rb
CHANGED
@@ -1,36 +1,28 @@
|
|
1
1
|
require 'celluloid'
|
2
2
|
require 'celluloid/autostart'
|
3
|
+
require 'sqskiq/signal_handler'
|
3
4
|
|
4
5
|
module Sqskiq
|
5
6
|
class Processor
|
6
7
|
include Celluloid
|
7
|
-
include
|
8
|
+
include Sqskiq::SignalHandler
|
8
9
|
|
9
10
|
def initialize(worker_class)
|
10
11
|
@worker_instance = worker_class.new
|
11
|
-
|
12
|
-
subscribe_interrupt
|
12
|
+
subscribe_for_shutdown
|
13
13
|
end
|
14
14
|
|
15
15
|
def process(message)
|
16
|
+
return { :success => false, :message => message } if @shutting_down
|
17
|
+
|
16
18
|
result = true
|
17
19
|
begin
|
18
20
|
@worker_instance.perform(message)
|
19
21
|
rescue Exception => e
|
20
22
|
result = false
|
21
23
|
end
|
22
|
-
{ :success => result, :message => message }
|
23
|
-
end
|
24
|
-
|
25
|
-
def subscribe_interrupt
|
26
|
-
subscribe('SIGINT', :interrupt)
|
27
|
-
subscribe('TERM', :interrupt)
|
28
|
-
subscribe('SIGTERM', :interrupt)
|
24
|
+
return { :success => result, :message => message }
|
29
25
|
end
|
30
|
-
|
31
|
-
def interrupt(signal)
|
32
|
-
self.terminate
|
33
|
-
end
|
34
|
-
|
26
|
+
|
35
27
|
end
|
36
28
|
end
|
data/lib/sqskiq/worker.rb
CHANGED
data/lib/sqskiq.rb
CHANGED
@@ -8,28 +8,17 @@ require 'sqskiq/batch_process'
|
|
8
8
|
module Sqskiq
|
9
9
|
|
10
10
|
def self.bootstrap(options, worker_class)
|
11
|
-
|
12
11
|
params = [ @aws_access_key_id, @aws_secret_access_key, options[:queue_name] ]
|
13
|
-
|
12
|
+
|
13
|
+
configured_pool_sizes = pool_sizes(options)
|
14
|
+
|
14
15
|
Celluloid::Actor[:manager] = @manager = Manager.new
|
15
|
-
Celluloid::Actor[:fetcher] = @fetcher = Fetcher.pool(:size =>
|
16
|
-
Celluloid::Actor[:processor] = @processor = Processor.pool(:size =>
|
17
|
-
Celluloid::Actor[:batch_processor] = @batch_processor = BatchProcessor.pool(:size =>
|
18
|
-
Celluloid::Actor[:deleter] = @deleter = Deleter.pool(:size =>
|
19
|
-
|
20
|
-
p "pid = #{Process.pid}"
|
21
|
-
|
22
|
-
trap('SIGTERM') do
|
23
|
-
@manager.publish('SIGTERM')
|
24
|
-
end
|
25
|
-
|
26
|
-
trap('TERM') do
|
27
|
-
@manager.publish('TERM')
|
28
|
-
end
|
16
|
+
Celluloid::Actor[:fetcher] = @fetcher = Fetcher.pool(:size => configured_pool_sizes[:num_fetchers], :args => params)
|
17
|
+
Celluloid::Actor[:processor] = @processor = Processor.pool(:size => configured_pool_sizes[:num_workers], :args => worker_class)
|
18
|
+
Celluloid::Actor[:batch_processor] = @batch_processor = BatchProcessor.pool(:size => configured_pool_sizes[:num_batches])
|
19
|
+
Celluloid::Actor[:deleter] = @deleter = Deleter.pool(:size => configured_pool_sizes[:num_deleters], :args => params)
|
29
20
|
|
30
|
-
|
31
|
-
@manager.publish('SIGINT')
|
32
|
-
end
|
21
|
+
configure_signal_listeners
|
33
22
|
|
34
23
|
@manager.bootstrap
|
35
24
|
while @manager.running? do
|
@@ -43,7 +32,35 @@ module Sqskiq
|
|
43
32
|
|
44
33
|
@manager.terminate
|
45
34
|
end
|
46
|
-
|
35
|
+
|
36
|
+
def self.configure_signal_listeners
|
37
|
+
['SIGTERM', 'TERM', 'SIGINT'].each do |signal|
|
38
|
+
trap(signal) do
|
39
|
+
@manager.publish('SIGTERM')
|
40
|
+
@batch_processor.publish('SIGTERM')
|
41
|
+
@processor.publish('SIGTERM')
|
42
|
+
end
|
43
|
+
end
|
44
|
+
end
|
45
|
+
|
46
|
+
def self.pool_sizes(options)
|
47
|
+
# for now, min processors should be 2
|
48
|
+
num_workers = (options[:processors].nil? || options[:processors].to_i < 2)? 20 : options[:processors]
|
49
|
+
|
50
|
+
# each fetch brings up to 10 messages to process.
|
51
|
+
# the number of fetchers is a number able to keep all
|
52
|
+
# workers handling messages
|
53
|
+
# TODO: acctualy the min number must be greater than 2 because we are using
|
54
|
+
# celluloid pool, but that will be changed!
|
55
|
+
num_fetchers = num_workers / 10
|
56
|
+
num_fetchers = num_fetchers + 1 if num_workers % 10 > 0
|
57
|
+
num_fetchers = 2 if num_fetchers < 2
|
58
|
+
|
59
|
+
num_deleters = num_batches = num_fetchers
|
60
|
+
|
61
|
+
{ num_workers: num_workers, num_fetchers: num_fetchers, num_batches: num_batches, num_deleters: num_deleters }
|
62
|
+
end
|
63
|
+
|
47
64
|
def self.configure
|
48
65
|
yield self
|
49
66
|
end
|
@@ -0,0 +1,70 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
|
3
|
+
describe Sqskiq::Manager do
|
4
|
+
let(:deleter) { Object.new }
|
5
|
+
let(:fetcher) { Object.new }
|
6
|
+
let(:batch_processor) { Object.new }
|
7
|
+
let(:shutting_down) { false }
|
8
|
+
subject = described_class.new
|
9
|
+
|
10
|
+
before do
|
11
|
+
subject.instance_variable_set(:@fetcher, fetcher)
|
12
|
+
subject.instance_variable_set(:@deleter, deleter)
|
13
|
+
subject.instance_variable_set(:@batch_processor, batch_processor)
|
14
|
+
subject.instance_variable_set(:@shutting_down, shutting_down)
|
15
|
+
end
|
16
|
+
|
17
|
+
describe '#running?' do
|
18
|
+
|
19
|
+
context 'when the actor system is shutting down' do
|
20
|
+
let(:shutting_down) { true }
|
21
|
+
|
22
|
+
describe 'if deleter is not empty' do
|
23
|
+
before { deleter.should_receive(:busy_size).and_return(1) }
|
24
|
+
|
25
|
+
it 'returns true' do
|
26
|
+
subject.running?.should be_true
|
27
|
+
end
|
28
|
+
end
|
29
|
+
|
30
|
+
describe 'if batch_processor is not empty' do
|
31
|
+
before do
|
32
|
+
deleter.should_receive(:busy_size).and_return(0)
|
33
|
+
batch_processor.should_receive(:busy_size).and_return(1)
|
34
|
+
end
|
35
|
+
|
36
|
+
it 'returns true' do
|
37
|
+
subject.running?.should be_true
|
38
|
+
end
|
39
|
+
end
|
40
|
+
|
41
|
+
describe 'if batch_processor and deleter are empties' do
|
42
|
+
before do
|
43
|
+
deleter.should_receive(:busy_size).and_return(0)
|
44
|
+
batch_processor.should_receive(:busy_size).and_return(0)
|
45
|
+
end
|
46
|
+
|
47
|
+
it 'returns false' do
|
48
|
+
subject.running?.should be_false
|
49
|
+
end
|
50
|
+
end
|
51
|
+
|
52
|
+
end
|
53
|
+
|
54
|
+
context 'when the actor system is not shutting down' do
|
55
|
+
|
56
|
+
describe 'even if batch_processor and deleter are empties' do
|
57
|
+
before do
|
58
|
+
deleter.stub(:busy_size).and_return(0)
|
59
|
+
batch_processor.stub(:busy_size).and_return(0)
|
60
|
+
end
|
61
|
+
|
62
|
+
it 'returns true' do
|
63
|
+
subject.running?.should be_true
|
64
|
+
end
|
65
|
+
end
|
66
|
+
|
67
|
+
end
|
68
|
+
end
|
69
|
+
|
70
|
+
end
|
@@ -0,0 +1,14 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
|
3
|
+
describe Sqskiq::Processor do
|
4
|
+
let(:worker) { double }
|
5
|
+
subject { described_class.new double(new: worker) }
|
6
|
+
|
7
|
+
let(:message) { double(body: 'hello!') }
|
8
|
+
|
9
|
+
context "Error handling" do
|
10
|
+
before { worker.stub(:perform) { raise Exception.new "OMG!" } }
|
11
|
+
|
12
|
+
it { expect(subject.process(message)[:success]).to be_false }
|
13
|
+
end
|
14
|
+
end
|
data/spec/spec_helper.rb
ADDED
@@ -0,0 +1 @@
|
|
1
|
+
require 'sqskiq'
|
data/spec/sqskiq_spec.rb
ADDED
@@ -0,0 +1,60 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
|
3
|
+
describe Sqskiq do
|
4
|
+
|
5
|
+
describe 'number of processors is lesser than 2' do
|
6
|
+
let(:options) { [ { processors: 1 }, ].sample }
|
7
|
+
|
8
|
+
it 'uses the defaut value of 20' do
|
9
|
+
pool_sizes = Sqskiq.pool_sizes(options)
|
10
|
+
pool_sizes[:num_workers].should eq(20)
|
11
|
+
pool_sizes[:num_fetchers].should eq(2)
|
12
|
+
pool_sizes[:num_batches].should eq(2)
|
13
|
+
pool_sizes[:num_deleters].should eq(2)
|
14
|
+
end
|
15
|
+
end
|
16
|
+
|
17
|
+
describe 'number of processor is greater than 2' do
|
18
|
+
|
19
|
+
describe 'with nothing remaining after performing division by 10' do
|
20
|
+
let(:options) { { processors: [ 20, 30, 40 ].sample } }
|
21
|
+
|
22
|
+
it 'uses the the given value' do
|
23
|
+
pool_sizes = Sqskiq.pool_sizes(options)
|
24
|
+
pool_sizes[:num_workers].should eq(options[:processors])
|
25
|
+
pool_sizes[:num_fetchers].should eq(options[:processors] / 10)
|
26
|
+
pool_sizes[:num_batches].should eq(options[:processors] / 10)
|
27
|
+
pool_sizes[:num_deleters].should eq(options[:processors] / 10)
|
28
|
+
end
|
29
|
+
|
30
|
+
end
|
31
|
+
|
32
|
+
describe 'with remaining value after performing division by 10' do
|
33
|
+
let(:options) { { processors: [ 21, 31, 41 ].sample } }
|
34
|
+
|
35
|
+
it 'uses the the given value for the processors and apply (processors / 10) + 1 for other pool sizes' do
|
36
|
+
pool_sizes = Sqskiq.pool_sizes(options)
|
37
|
+
pool_sizes[:num_workers].should eq(options[:processors])
|
38
|
+
pool_sizes[:num_fetchers].should eq((options[:processors] / 10) + 1)
|
39
|
+
pool_sizes[:num_batches].should eq((options[:processors] / 10) + 1)
|
40
|
+
pool_sizes[:num_deleters].should eq((options[:processors] / 10) + 1)
|
41
|
+
end
|
42
|
+
|
43
|
+
end
|
44
|
+
|
45
|
+
describe 'and lesser or equals to 10' do
|
46
|
+
let(:options) { { processors: Random.rand(2..10) } }
|
47
|
+
|
48
|
+
it 'uses the the given value for the processors and apply (processors / 10) for other pool sizes' do
|
49
|
+
pool_sizes = Sqskiq.pool_sizes(options)
|
50
|
+
pool_sizes[:num_workers].should eq(options[:processors])
|
51
|
+
pool_sizes[:num_fetchers].should eq((options[:processors] / 10) + 2)
|
52
|
+
pool_sizes[:num_batches].should eq((options[:processors] / 10) + 2)
|
53
|
+
pool_sizes[:num_deleters].should eq((options[:processors] / 10) + 2)
|
54
|
+
end
|
55
|
+
|
56
|
+
end
|
57
|
+
end
|
58
|
+
|
59
|
+
|
60
|
+
end
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: sqskiq
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.0.
|
4
|
+
version: 0.0.2
|
5
5
|
prerelease:
|
6
6
|
platform: ruby
|
7
7
|
authors:
|
@@ -43,6 +43,38 @@ dependencies:
|
|
43
43
|
- - ~>
|
44
44
|
- !ruby/object:Gem::Version
|
45
45
|
version: 1.9.1
|
46
|
+
- !ruby/object:Gem::Dependency
|
47
|
+
name: activesupport
|
48
|
+
requirement: !ruby/object:Gem::Requirement
|
49
|
+
none: false
|
50
|
+
requirements:
|
51
|
+
- - ! '>='
|
52
|
+
- !ruby/object:Gem::Version
|
53
|
+
version: '0'
|
54
|
+
type: :runtime
|
55
|
+
prerelease: false
|
56
|
+
version_requirements: !ruby/object:Gem::Requirement
|
57
|
+
none: false
|
58
|
+
requirements:
|
59
|
+
- - ! '>='
|
60
|
+
- !ruby/object:Gem::Version
|
61
|
+
version: '0'
|
62
|
+
- !ruby/object:Gem::Dependency
|
63
|
+
name: rspec
|
64
|
+
requirement: !ruby/object:Gem::Requirement
|
65
|
+
none: false
|
66
|
+
requirements:
|
67
|
+
- - ! '>='
|
68
|
+
- !ruby/object:Gem::Version
|
69
|
+
version: '0'
|
70
|
+
type: :development
|
71
|
+
prerelease: false
|
72
|
+
version_requirements: !ruby/object:Gem::Requirement
|
73
|
+
none: false
|
74
|
+
requirements:
|
75
|
+
- - ! '>='
|
76
|
+
- !ruby/object:Gem::Version
|
77
|
+
version: '0'
|
46
78
|
description: sqskiq
|
47
79
|
email: ri.vanlazar@gmail.com
|
48
80
|
executables: []
|
@@ -57,6 +89,10 @@ files:
|
|
57
89
|
- lib/sqskiq/worker.rb
|
58
90
|
- lib/sqskiq/batch_process.rb
|
59
91
|
- lib/sqskiq/aws.rb
|
92
|
+
- spec/manager_spec.rb
|
93
|
+
- spec/processor_spec.rb
|
94
|
+
- spec/spec_helper.rb
|
95
|
+
- spec/sqskiq_spec.rb
|
60
96
|
homepage: http://rubygems.org/gems/sqskiq
|
61
97
|
licenses: []
|
62
98
|
post_install_message:
|
@@ -81,4 +117,8 @@ rubygems_version: 1.8.25
|
|
81
117
|
signing_key:
|
82
118
|
specification_version: 3
|
83
119
|
summary: sqskiq
|
84
|
-
test_files:
|
120
|
+
test_files:
|
121
|
+
- spec/manager_spec.rb
|
122
|
+
- spec/processor_spec.rb
|
123
|
+
- spec/spec_helper.rb
|
124
|
+
- spec/sqskiq_spec.rb
|