sqskiq 0.0.1
Sign up to get free protection for your applications and to get access to all the features.
- data/lib/sqskiq/aws.rb +20 -0
- data/lib/sqskiq/batch_process.rb +46 -0
- data/lib/sqskiq/delete.rb +19 -0
- data/lib/sqskiq/fetch.rb +34 -0
- data/lib/sqskiq/manager.rb +50 -0
- data/lib/sqskiq/process.rb +36 -0
- data/lib/sqskiq/worker.rb +21 -0
- data/lib/sqskiq.rb +59 -0
- metadata +84 -0
data/lib/sqskiq/aws.rb
ADDED
@@ -0,0 +1,20 @@
|
|
1
|
+
require 'aws-sdk'
|
2
|
+
|
3
|
+
module Sqskiq
|
4
|
+
module AWS
|
5
|
+
|
6
|
+
def init_queue(aws_access_key_id, aws_secret_access_key, queue_name)
|
7
|
+
sqs = ::AWS::SQS.new(:access_key_id => aws_access_key_id, :secret_access_key => aws_secret_access_key)
|
8
|
+
@queue = sqs.queues.named(queue_name.to_s)
|
9
|
+
end
|
10
|
+
|
11
|
+
def fetch_sqs_messages
|
12
|
+
@queue.receive_message(:limit => 10, :attributes => [:receive_count])
|
13
|
+
end
|
14
|
+
|
15
|
+
def delete_sqs_messages(messages)
|
16
|
+
@queue.batch_delete(messages)
|
17
|
+
end
|
18
|
+
|
19
|
+
end
|
20
|
+
end
|
@@ -0,0 +1,46 @@
|
|
1
|
+
require 'celluloid'
|
2
|
+
require 'celluloid/autostart'
|
3
|
+
|
4
|
+
module Sqskiq
|
5
|
+
class BatchProcessor
|
6
|
+
include Celluloid
|
7
|
+
include Celluloid::Notifications
|
8
|
+
|
9
|
+
def initialize
|
10
|
+
@manager = Celluloid::Actor[:manager]
|
11
|
+
@processor = Celluloid::Actor[:processor]
|
12
|
+
|
13
|
+
subscribe_interrupt
|
14
|
+
end
|
15
|
+
|
16
|
+
def batch_process(messages)
|
17
|
+
p "processing #{messages.size} messages"
|
18
|
+
|
19
|
+
process_result = []
|
20
|
+
messages.each do |message|
|
21
|
+
process_result << @processor.future.process(message)
|
22
|
+
end
|
23
|
+
|
24
|
+
success_messages = []
|
25
|
+
process_result.each do |result|
|
26
|
+
value = result.value
|
27
|
+
if value[:success]
|
28
|
+
success_messages << value[:message]
|
29
|
+
end
|
30
|
+
end
|
31
|
+
|
32
|
+
@manager.async.batch_process_done(success_messages)
|
33
|
+
end
|
34
|
+
|
35
|
+
def subscribe_interrupt
|
36
|
+
subscribe('SIGINT', :interrupt)
|
37
|
+
subscribe('TERM', :interrupt)
|
38
|
+
subscribe('SIGTERM', :interrupt)
|
39
|
+
end
|
40
|
+
|
41
|
+
def interrupt(signal)
|
42
|
+
self.terminate
|
43
|
+
end
|
44
|
+
|
45
|
+
end
|
46
|
+
end
|
@@ -0,0 +1,19 @@
|
|
1
|
+
require 'celluloid'
|
2
|
+
require 'celluloid/autostart'
|
3
|
+
require 'sqskiq/aws'
|
4
|
+
|
5
|
+
module Sqskiq
|
6
|
+
class Deleter
|
7
|
+
include Celluloid
|
8
|
+
include Celluloid::Notifications
|
9
|
+
include Sqskiq::AWS
|
10
|
+
|
11
|
+
def initialize(aws_access_key_id, aws_secret_access_key, queue_name)
|
12
|
+
init_queue(aws_access_key_id, aws_secret_access_key, queue_name)
|
13
|
+
end
|
14
|
+
|
15
|
+
def delete(messages)
|
16
|
+
delete_sqs_messages(messages) if not messages.empty?
|
17
|
+
end
|
18
|
+
end
|
19
|
+
end
|
data/lib/sqskiq/fetch.rb
ADDED
@@ -0,0 +1,34 @@
|
|
1
|
+
require 'celluloid'
|
2
|
+
require 'celluloid/autostart'
|
3
|
+
require 'sqskiq/aws'
|
4
|
+
|
5
|
+
module Sqskiq
|
6
|
+
class Fetcher
|
7
|
+
include Celluloid
|
8
|
+
include Celluloid::Notifications
|
9
|
+
include Sqskiq::AWS
|
10
|
+
|
11
|
+
def initialize(aws_access_key_id, aws_secret_access_key, queue_name)
|
12
|
+
init_queue(aws_access_key_id, aws_secret_access_key, queue_name)
|
13
|
+
subscribe_interrupt
|
14
|
+
|
15
|
+
@manager = Celluloid::Actor[:manager]
|
16
|
+
end
|
17
|
+
|
18
|
+
def fetch
|
19
|
+
messages = fetch_sqs_messages
|
20
|
+
@manager.async.fetch_done(messages)
|
21
|
+
end
|
22
|
+
|
23
|
+
def subscribe_interrupt
|
24
|
+
subscribe('SIGINT', :interrupt)
|
25
|
+
subscribe('SIGTERM', :interrupt)
|
26
|
+
subscribe('TERM', :interrupt)
|
27
|
+
end
|
28
|
+
|
29
|
+
def interrupt(signal)
|
30
|
+
self.terminate
|
31
|
+
end
|
32
|
+
|
33
|
+
end
|
34
|
+
end
|
@@ -0,0 +1,50 @@
|
|
1
|
+
require 'celluloid'
|
2
|
+
require 'celluloid/autostart'
|
3
|
+
|
4
|
+
|
5
|
+
module Sqskiq
|
6
|
+
class Manager
|
7
|
+
include Celluloid
|
8
|
+
include Celluloid::Notifications
|
9
|
+
|
10
|
+
def initialize
|
11
|
+
@shutting_down = false
|
12
|
+
subscribe_shutting_down
|
13
|
+
end
|
14
|
+
|
15
|
+
def bootstrap
|
16
|
+
@fetcher = Celluloid::Actor[:fetcher]
|
17
|
+
@batch_processor = Celluloid::Actor[:batch_processor]
|
18
|
+
@deleter = Celluloid::Actor[:deleter]
|
19
|
+
#TODO default value. Should be configurable
|
20
|
+
new_fetch(2)
|
21
|
+
end
|
22
|
+
|
23
|
+
def fetch_done(messages)
|
24
|
+
@batch_processor.async.batch_process(messages) unless @shutting_down
|
25
|
+
end
|
26
|
+
|
27
|
+
def batch_process_done(messages)
|
28
|
+
@deleter.async.delete(messages) if @deleter.alive?
|
29
|
+
new_fetch(1)
|
30
|
+
end
|
31
|
+
|
32
|
+
def new_fetch(num)
|
33
|
+
num.times { @fetcher.async.fetch unless @shutting_down }
|
34
|
+
end
|
35
|
+
|
36
|
+
def shutting_down(signal)
|
37
|
+
@shutting_down = true
|
38
|
+
end
|
39
|
+
|
40
|
+
def running?
|
41
|
+
not (@deleter.busy_size == 0 and @shutting_down)
|
42
|
+
end
|
43
|
+
|
44
|
+
def subscribe_shutting_down
|
45
|
+
subscribe('SIGINT', :shutting_down)
|
46
|
+
subscribe('TERM', :shutting_down)
|
47
|
+
subscribe('SIGTERM', :shutting_down)
|
48
|
+
end
|
49
|
+
end
|
50
|
+
end
|
@@ -0,0 +1,36 @@
|
|
1
|
+
require 'celluloid'
|
2
|
+
require 'celluloid/autostart'
|
3
|
+
|
4
|
+
module Sqskiq
|
5
|
+
class Processor
|
6
|
+
include Celluloid
|
7
|
+
include Celluloid::Notifications
|
8
|
+
|
9
|
+
def initialize(worker_class)
|
10
|
+
@worker_instance = worker_class.new
|
11
|
+
|
12
|
+
subscribe_interrupt
|
13
|
+
end
|
14
|
+
|
15
|
+
def process(message)
|
16
|
+
result = true
|
17
|
+
begin
|
18
|
+
@worker_instance.perform(message)
|
19
|
+
rescue Exception => e
|
20
|
+
result = false
|
21
|
+
end
|
22
|
+
{ :success => result, :message => message }
|
23
|
+
end
|
24
|
+
|
25
|
+
def subscribe_interrupt
|
26
|
+
subscribe('SIGINT', :interrupt)
|
27
|
+
subscribe('TERM', :interrupt)
|
28
|
+
subscribe('SIGTERM', :interrupt)
|
29
|
+
end
|
30
|
+
|
31
|
+
def interrupt(signal)
|
32
|
+
self.terminate
|
33
|
+
end
|
34
|
+
|
35
|
+
end
|
36
|
+
end
|
@@ -0,0 +1,21 @@
|
|
1
|
+
module Sqskiq
|
2
|
+
module Worker
|
3
|
+
|
4
|
+
module ClassMethods
|
5
|
+
def run
|
6
|
+
Sqskiq.bootstrap(sqskiq_options_hash, self)
|
7
|
+
end
|
8
|
+
|
9
|
+
def sqskiq_options(options)
|
10
|
+
self.sqskiq_options_hash = options
|
11
|
+
end
|
12
|
+
end
|
13
|
+
|
14
|
+
def self.included(base)
|
15
|
+
base.send :extend, ClassMethods
|
16
|
+
base.class_attribute :sqskiq_options_hash
|
17
|
+
end
|
18
|
+
|
19
|
+
end
|
20
|
+
|
21
|
+
end
|
data/lib/sqskiq.rb
ADDED
@@ -0,0 +1,59 @@
|
|
1
|
+
require "sqskiq/manager"
|
2
|
+
require 'sqskiq/fetch'
|
3
|
+
require 'sqskiq/process'
|
4
|
+
require 'sqskiq/delete'
|
5
|
+
require 'sqskiq/worker'
|
6
|
+
require 'sqskiq/batch_process'
|
7
|
+
|
8
|
+
module Sqskiq
|
9
|
+
|
10
|
+
def self.bootstrap(options, worker_class)
|
11
|
+
|
12
|
+
params = [ @aws_access_key_id, @aws_secret_access_key, options[:queue_name] ]
|
13
|
+
|
14
|
+
Celluloid::Actor[:manager] = @manager = Manager.new
|
15
|
+
Celluloid::Actor[:fetcher] = @fetcher = Fetcher.pool(:size => 2, :args => params)
|
16
|
+
Celluloid::Actor[:processor] = @processor = Processor.pool(:size => 20, :args => worker_class)
|
17
|
+
Celluloid::Actor[:batch_processor] = @batch_processor = BatchProcessor.pool(:size => 2)
|
18
|
+
Celluloid::Actor[:deleter] = @deleter = Deleter.pool(:size => 2, :args => params)
|
19
|
+
|
20
|
+
p "pid = #{Process.pid}"
|
21
|
+
|
22
|
+
trap('SIGTERM') do
|
23
|
+
@manager.publish('SIGTERM')
|
24
|
+
end
|
25
|
+
|
26
|
+
trap('TERM') do
|
27
|
+
@manager.publish('TERM')
|
28
|
+
end
|
29
|
+
|
30
|
+
trap('SIGINT') do
|
31
|
+
@manager.publish('SIGINT')
|
32
|
+
end
|
33
|
+
|
34
|
+
@manager.bootstrap
|
35
|
+
while @manager.running? do
|
36
|
+
sleep 2
|
37
|
+
end
|
38
|
+
|
39
|
+
@fetcher.__shutdown__
|
40
|
+
@batch_processor.__shutdown__
|
41
|
+
@processor.__shutdown__
|
42
|
+
@deleter.__shutdown__
|
43
|
+
|
44
|
+
@manager.terminate
|
45
|
+
end
|
46
|
+
|
47
|
+
def self.configure
|
48
|
+
yield self
|
49
|
+
end
|
50
|
+
|
51
|
+
def self.aws_access_key_id=(value)
|
52
|
+
@aws_access_key_id = value
|
53
|
+
end
|
54
|
+
|
55
|
+
def self.aws_secret_access_key=(value)
|
56
|
+
@aws_secret_access_key = value
|
57
|
+
end
|
58
|
+
|
59
|
+
end
|
metadata
ADDED
@@ -0,0 +1,84 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: sqskiq
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 0.0.1
|
5
|
+
prerelease:
|
6
|
+
platform: ruby
|
7
|
+
authors:
|
8
|
+
- Ricardo Lazaro de Oliveira
|
9
|
+
autorequire:
|
10
|
+
bindir: bin
|
11
|
+
cert_chain: []
|
12
|
+
date: 2013-06-01 00:00:00.000000000 Z
|
13
|
+
dependencies:
|
14
|
+
- !ruby/object:Gem::Dependency
|
15
|
+
name: celluloid
|
16
|
+
requirement: !ruby/object:Gem::Requirement
|
17
|
+
none: false
|
18
|
+
requirements:
|
19
|
+
- - ~>
|
20
|
+
- !ruby/object:Gem::Version
|
21
|
+
version: 0.14.1
|
22
|
+
type: :runtime
|
23
|
+
prerelease: false
|
24
|
+
version_requirements: !ruby/object:Gem::Requirement
|
25
|
+
none: false
|
26
|
+
requirements:
|
27
|
+
- - ~>
|
28
|
+
- !ruby/object:Gem::Version
|
29
|
+
version: 0.14.1
|
30
|
+
- !ruby/object:Gem::Dependency
|
31
|
+
name: aws-sdk
|
32
|
+
requirement: !ruby/object:Gem::Requirement
|
33
|
+
none: false
|
34
|
+
requirements:
|
35
|
+
- - ~>
|
36
|
+
- !ruby/object:Gem::Version
|
37
|
+
version: 1.9.1
|
38
|
+
type: :runtime
|
39
|
+
prerelease: false
|
40
|
+
version_requirements: !ruby/object:Gem::Requirement
|
41
|
+
none: false
|
42
|
+
requirements:
|
43
|
+
- - ~>
|
44
|
+
- !ruby/object:Gem::Version
|
45
|
+
version: 1.9.1
|
46
|
+
description: sqskiq
|
47
|
+
email: ri.vanlazar@gmail.com
|
48
|
+
executables: []
|
49
|
+
extensions: []
|
50
|
+
extra_rdoc_files: []
|
51
|
+
files:
|
52
|
+
- lib/sqskiq.rb
|
53
|
+
- lib/sqskiq/manager.rb
|
54
|
+
- lib/sqskiq/fetch.rb
|
55
|
+
- lib/sqskiq/process.rb
|
56
|
+
- lib/sqskiq/delete.rb
|
57
|
+
- lib/sqskiq/worker.rb
|
58
|
+
- lib/sqskiq/batch_process.rb
|
59
|
+
- lib/sqskiq/aws.rb
|
60
|
+
homepage: http://rubygems.org/gems/sqskiq
|
61
|
+
licenses: []
|
62
|
+
post_install_message:
|
63
|
+
rdoc_options: []
|
64
|
+
require_paths:
|
65
|
+
- lib
|
66
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
67
|
+
none: false
|
68
|
+
requirements:
|
69
|
+
- - ! '>='
|
70
|
+
- !ruby/object:Gem::Version
|
71
|
+
version: '0'
|
72
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
73
|
+
none: false
|
74
|
+
requirements:
|
75
|
+
- - ! '>='
|
76
|
+
- !ruby/object:Gem::Version
|
77
|
+
version: '0'
|
78
|
+
requirements: []
|
79
|
+
rubyforge_project:
|
80
|
+
rubygems_version: 1.8.25
|
81
|
+
signing_key:
|
82
|
+
specification_version: 3
|
83
|
+
summary: sqskiq
|
84
|
+
test_files: []
|